feldera 0.148.0__tar.gz → 0.149.0__tar.gz

{feldera-0.148.0 → feldera-0.149.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.148.0
+Version: 0.149.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

{feldera-0.148.0 → feldera-0.149.0}/feldera/_callback_runner.py

@@ -6,6 +6,7 @@ from queue import Queue, Empty
 import pandas as pd
 from feldera import FelderaClient
 from feldera._helpers import dataframe_from_response
+from feldera.enums import PipelineFieldSelector
 
 
 class _CallbackRunnerInstruction(Enum):
@@ -38,7 +39,9 @@ class CallbackRunner(Thread):
         :meta private:
         """
 
-        pipeline = self.client.get_pipeline(self.pipeline_name)
+        pipeline = self.client.get_pipeline(
+            self.pipeline_name, PipelineFieldSelector.ALL
+        )
 
         schemas = pipeline.tables + pipeline.views
         for schema in schemas:

{feldera-0.148.0 → feldera-0.149.0}/feldera/enums.py

@@ -336,3 +336,11 @@ class FaultToleranceModel(Enum):
         raise ValueError(
             f"Unknown value '{value}' for enum {FaultToleranceModel.__name__}"
         )
+
+
+class PipelineFieldSelector(Enum):
+    ALL = "all"
+    """Select all fields of a pipeline."""
+
+    STATUS = "status"
+    """Select only the fields required to know the status of a pipeline."""

{feldera-0.148.0 → feldera-0.149.0}/feldera/pipeline.py

@@ -12,6 +12,7 @@ from queue import Queue
 
 from feldera.rest.errors import FelderaAPIError
 from feldera.enums import (
+    PipelineFieldSelector,
     PipelineStatus,
     ProgramStatus,
     CheckpointStatus,
@@ -60,14 +61,16 @@ class Pipeline:
                 # block until the callback runner is ready
                 queue.join()
 
-    def refresh(self):
+    def refresh(self, field_selector: PipelineFieldSelector):
         """
         Calls the backend to get the updated, latest version of the pipeline.
 
+        :param field_selector: Choose what pipeline information to refresh; see PipelineFieldSelector enum definition.
+
         :raises FelderaConnectionError: If there is an issue connecting to the backend.
         """
 
-        self._inner = self.client.get_pipeline(self.name)
+        self._inner = self.client.get_pipeline(self.name, field_selector)
 
     def status(self) -> PipelineStatus:
         """
@@ -75,7 +78,7 @@ class Pipeline:
         """
 
         try:
-            self.refresh()
+            self.refresh(PipelineFieldSelector.STATUS)
             return PipelineStatus.from_str(self._inner.deployment_status)
 
         except FelderaAPIError as err:
@@ -124,7 +127,7 @@ class Pipeline:
 
         ensure_dataframe_has_columns(df)
 
-        pipeline = self.client.get_pipeline(self.name)
+        pipeline = self.client.get_pipeline(self.name, PipelineFieldSelector.ALL)
         if table_name.lower() != "now" and table_name.lower() not in [
             tbl.name.lower() for tbl in pipeline.tables
         ]:
@@ -672,7 +675,7 @@ metrics"""
         """
 
         try:
-            inner = client.get_pipeline(name)
+            inner = client.get_pipeline(name, PipelineFieldSelector.ALL)
             return Pipeline._from_inner(inner, client)
         except FelderaAPIError as err:
             if err.status_code == 404:
@@ -948,7 +951,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the program SQL code of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.program_code
 
     def modify(
@@ -988,7 +991,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the storage status of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return StorageStatus.from_str(self._inner.storage_status)
 
     def program_status(self) -> ProgramStatus:
@@ -1000,7 +1003,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Rust code to a binary.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return ProgramStatus.from_value(self._inner.program_status)
 
     def program_status_since(self) -> datetime:
@@ -1008,7 +1011,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the timestamp when the current program status was set.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return datetime.fromisoformat(self._inner.program_status_since)
 
     def udf_rust(self) -> str:
@@ -1016,7 +1019,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the Rust code for UDFs.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.udf_rust
 
     def udf_toml(self) -> str:
@@ -1024,7 +1027,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the Rust dependencies required by UDFs (in the TOML format).
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.udf_toml
 
     def program_config(self) -> Mapping[str, Any]:
@@ -1032,7 +1035,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the program config of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.program_config
 
     def runtime_config(self) -> RuntimeConfig:
@@ -1040,7 +1043,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the runtime config of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return RuntimeConfig.from_dict(self._inner.runtime_config)
 
     def set_runtime_config(self, runtime_config: RuntimeConfig):
@@ -1065,7 +1068,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the ID of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.id
 
     def description(self) -> str:
@@ -1073,7 +1076,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the description of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.description
 
     def tables(self) -> List[SQLTable]:
@@ -1081,7 +1084,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the tables of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.tables
 
     def views(self) -> List[SQLView]:
@@ -1089,7 +1092,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the views of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.views
 
     def created_at(self) -> datetime:
@@ -1097,7 +1100,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the creation time of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return datetime.fromisoformat(self._inner.created_at)
 
     def version(self) -> int:
@@ -1105,7 +1108,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the version of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.version
 
     def program_version(self) -> int:
@@ -1113,7 +1116,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the program version of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.program_version
 
     def deployment_status_since(self) -> datetime:
@@ -1122,7 +1125,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         was set.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return datetime.fromisoformat(self._inner.deployment_status_since)
 
     def deployment_config(self) -> Mapping[str, Any]:
@@ -1130,7 +1133,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the deployment config of the pipeline.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.deployment_config
 
     def deployment_desired_status(self) -> DeploymentDesiredStatus:
@@ -1139,7 +1142,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         This is the next state that the pipeline should transition to.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return DeploymentDesiredStatus.from_str(self._inner.deployment_desired_status)
 
     def deployment_resources_desired_status(self) -> DeploymentResourcesDesiredStatus:
@@ -1147,7 +1150,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the desired status of the the deployment resources.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return DeploymentResourcesDesiredStatus.from_str(
             self._inner.deployment_resources_desired_status
         )
@@ -1157,7 +1160,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the status of the deployment resources.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return DeploymentResourcesStatus.from_str(
             self._inner.deployment_resources_status
         )
@@ -1167,7 +1170,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the deployment runtime desired status.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return DeploymentRuntimeDesiredStatus.from_str(
             self._inner.deployment_runtime_desired_status
         )
@@ -1177,7 +1180,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Return the deployment runtime status.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return DeploymentRuntimeStatus.from_str(self._inner.deployment_runtime_status)
 
     def deployment_error(self) -> Mapping[str, Any]:
@@ -1186,7 +1189,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         Returns an empty string if there is no error.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.deployment_error
 
     def deployment_location(self) -> str:
@@ -1196,7 +1199,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         at runtime (a TCP port number or a URI).
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.STATUS)
         return self._inner.deployment_location
 
     def program_info(self) -> Mapping[str, Any]:
@@ -1207,7 +1210,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         and the SQL program schema.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.program_info
 
     def program_error(self) -> Mapping[str, Any]:
@@ -1217,7 +1220,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         `sql_compilation` and `rust_compilation` will be 0.
         """
 
-        self.refresh()
+        self.refresh(PipelineFieldSelector.ALL)
         return self._inner.program_error
 
     def errors(self) -> List[Mapping[str, Any]]:

{feldera-0.148.0 → feldera-0.149.0}/feldera/pipeline_builder.py

@@ -4,7 +4,7 @@ from typing import Optional
 from feldera.rest.feldera_client import FelderaClient
 from feldera.rest.pipeline import Pipeline as InnerPipeline
 from feldera.pipeline import Pipeline
-from feldera.enums import CompilationProfile
+from feldera.enums import CompilationProfile, PipelineFieldSelector
 from feldera.runtime_config import RuntimeConfig
 from feldera.rest.errors import FelderaAPIError
 
@@ -60,7 +60,10 @@ class PipelineBuilder:
             raise ValueError("Name and SQL are required to create a pipeline")
 
         try:
-            if self.client.get_pipeline(self.name) is not None:
+            if (
+                self.client.get_pipeline(self.name, PipelineFieldSelector.STATUS)
+                is not None
+            ):
                 raise RuntimeError(f"Pipeline with name {self.name} already exists")
         except FelderaAPIError as err:
             if err.error_code != "UnknownPipelineName":

{feldera-0.148.0 → feldera-0.149.0}/feldera/rest/feldera_client.py

@@ -7,6 +7,7 @@ from decimal import Decimal
 from typing import Generator, Mapping
 from urllib.parse import quote
 
+from feldera.enums import PipelineFieldSelector
 from feldera.rest.config import Config
 from feldera.rest.feldera_config import FelderaConfig
 from feldera.rest.errors import FelderaTimeoutError, FelderaAPIError
@@ -93,14 +94,19 @@ class FelderaClient:
 
         return FelderaClient(f"http://127.0.0.1:{port}")
 
-    def get_pipeline(self, pipeline_name) -> Pipeline:
+    def get_pipeline(
+        self, pipeline_name: str, field_selector: PipelineFieldSelector
+    ) -> Pipeline:
         """
         Get a pipeline by name
 
         :param pipeline_name: The name of the pipeline
+        :param field_selector: Choose what pipeline information to refresh; see PipelineFieldSelector enum definition.
         """
 
-        resp = self.http.get(f"/pipelines/{pipeline_name}")
+        resp = self.http.get(
+            f"/pipelines/{pipeline_name}?selector={field_selector.value}"
+        )
 
         return Pipeline.from_dict(resp)
 
@@ -130,12 +136,14 @@ class FelderaClient:
         wait = ["Pending", "CompilingSql", "SqlCompiled", "CompilingRust"]
 
         while True:
-            p = self.get_pipeline(name)
+            p = self.get_pipeline(name, PipelineFieldSelector.STATUS)
             status = p.program_status
 
             if status == "Success":
-                return p
+                return self.get_pipeline(name, PipelineFieldSelector.ALL)
             elif status not in wait:
+                p = self.get_pipeline(name, PipelineFieldSelector.ALL)
+
                 # error handling for SQL compilation errors
                 if status == "SqlError":
                     sql_errors = p.program_error["sql_compilation"]["messages"]
@@ -181,7 +189,7 @@ class FelderaClient:
                         f"transition to '{state}' state"
                     )
 
-            resp = self.get_pipeline(pipeline_name)
+            resp = self.get_pipeline(pipeline_name, PipelineFieldSelector.STATUS)
             status = resp.deployment_status
 
             if status.lower() == state.lower():
@@ -502,7 +510,9 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         start = time.monotonic()
 
         while time.monotonic() - start < timeout_s:
-            status = self.get_pipeline(pipeline_name).deployment_status
+            status = self.get_pipeline(
+                pipeline_name, PipelineFieldSelector.STATUS
+            ).deployment_status
 
             if status == "Stopped":
                 return
@@ -536,7 +546,9 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         start = time.monotonic()
 
         while time.monotonic() - start < timeout_s:
-            status = self.get_pipeline(pipeline_name).storage_status
+            status = self.get_pipeline(
+                pipeline_name, PipelineFieldSelector.STATUS
+            ).storage_status
 
             if status == "Cleared":
                 return

{feldera-0.148.0 → feldera-0.149.0}/feldera.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.148.0
+Version: 0.149.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

{feldera-0.148.0 → feldera-0.149.0}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "feldera"
 readme = "README.md"
 description = "The feldera python client"
-version = "0.148.0"
+version = "0.149.0"
 license = { text = "MIT" }
 requires-python = ">=3.10"
 authors = [