feldera 0.108.0__py3-none-any.whl → 0.110.0__py3-none-any.whl

feldera/pipeline.py

@@ -19,6 +19,7 @@ from feldera.output_handler import OutputHandler
 from feldera._helpers import ensure_dataframe_has_columns, chunk_dataframe
 from feldera.rest.sql_table import SQLTable
 from feldera.rest.sql_view import SQLView
+from feldera.runtime_config import RuntimeConfig
 from feldera.stats import PipelineStatistics
 
 
@@ -827,13 +828,30 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         self.refresh()
         return self._inner.program_config
 
-    def runtime_config(self) -> Mapping[str, Any]:
+    def runtime_config(self) -> RuntimeConfig:
         """
         Return the runtime config of the pipeline.
         """
 
         self.refresh()
-        return self._inner.runtime_config
+        return RuntimeConfig.from_dict(self._inner.runtime_config)
+
+    def set_runtime_config(self, runtime_config: RuntimeConfig):
+        """Updates the runtime config of the pipeline.  The pipeline
+        must be stopped and, in addition, changing some pipeline
+        configuration requires storage to be cleared.
+
+        For example, to set 'min_batch_size_records' on a pipeline:
+
+        runtime_config = pipeline.runtime_config()
+        runtime_config.min_batch_size_records = 500
+        pipeline.set_runtime_config(runtime_config)
+
+        """
+
+        self.client.patch_pipeline(
+            name=self._inner.name, runtime_config=runtime_config.to_dict()
+        )
 
     def id(self) -> str:
         """

feldera/pipeline_builder.py

@@ -1,3 +1,6 @@
+import os
+from typing import Optional
+
 from feldera.rest.feldera_client import FelderaClient
 from feldera.rest.pipeline import Pipeline as InnerPipeline
 from feldera.pipeline import Pipeline
@@ -32,6 +35,7 @@ class PipelineBuilder:
         description: str = "",
         compilation_profile: CompilationProfile = CompilationProfile.OPTIMIZED,
         runtime_config: RuntimeConfig = RuntimeConfig.default(),
+        runtime_version: Optional[str] = None,
     ):
         self.client: FelderaClient = client
         self.name: str | None = name
@@ -41,6 +45,9 @@ class PipelineBuilder:
         self.udf_toml: str = udf_toml
         self.compilation_profile: CompilationProfile = compilation_profile
         self.runtime_config: RuntimeConfig = runtime_config
+        self.runtime_version: Optional[str] = runtime_version or os.environ.get(
+            "FELDERA_RUNTIME_VERSION"
+        )
 
     def create(self) -> Pipeline:
         """
@@ -67,10 +74,9 @@ class PipelineBuilder:
             udf_toml=self.udf_toml,
             program_config={
                 "profile": self.compilation_profile.value,
+                "runtime_version": self.runtime_version,
             },
-            runtime_config=dict(
-                (k, v) for k, v in self.runtime_config.__dict__.items() if v is not None
-            ),
+            runtime_config=self.runtime_config.to_dict(),
         )
 
         inner = self.client.create_pipeline(inner)
@@ -108,9 +114,7 @@ class PipelineBuilder:
             program_config={
                 "profile": self.compilation_profile.value,
             },
-            runtime_config=dict(
-                (k, v) for k, v in self.runtime_config.__dict__.items() if v is not None
-            ),
+            runtime_config=self.runtime_config.to_dict(),
         )
 
         inner = self.client.create_or_update_pipeline(inner)

feldera/rest/feldera_client.py

@@ -4,7 +4,7 @@ import logging
 import time
 import json
 from decimal import Decimal
-from typing import Generator
+from typing import Generator, Mapping
 
 from feldera.rest.config import Config
 from feldera.rest.feldera_config import FelderaConfig
@@ -191,18 +191,32 @@ class FelderaClient:
 
         return self.__wait_for_compilation(pipeline.name)
 
-    def patch_pipeline(self, name: str, sql: str):
+    def patch_pipeline(
+        self,
+        name: str,
+        sql: Optional[str] = None,
+        udf_rust: Optional[str] = None,
+        udf_toml: Optional[str] = None,
+        program_config: Optional[Mapping[str, Any]] = None,
+        runtime_config: Optional[Mapping[str, Any]] = None,
+        description: Optional[str] = None,
+    ):
         """
-        Incrementally update the pipeline SQL
+        Incrementally update pipeline
 
         :param name: The name of the pipeline
-        :param sql: The SQL snippet. Replaces the existing SQL code with this
-            one.
         """
 
         self.http.patch(
             path=f"/pipelines/{name}",
-            body={"program_code": sql},
+            body={
+                "program_code": sql,
+                "udf_rust": udf_rust,
+                "udf_toml": udf_toml,
+                "program_config": program_config,
+                "runtime_config": runtime_config,
+                "description": description,
+            },
         )
 
     def delete_pipeline(self, name: str):

feldera/runtime_config.py

@@ -1,4 +1,3 @@
-import os
 from typing import Optional, Any, Mapping
 from feldera.enums import FaultToleranceModel
 
@@ -79,7 +78,6 @@ class RuntimeConfig:
         clock_resolution_usecs: Optional[int] = None,
         provisioning_timeout_secs: Optional[int] = None,
         resources: Optional[Resources] = None,
-        runtime_version: Optional[str] = None,
         fault_tolerance_model: Optional[FaultToleranceModel] = None,
         checkpoint_interval_secs: Optional[int] = None,
     ):
@@ -91,9 +89,6 @@ class RuntimeConfig:
         self.min_batch_size_records = min_batch_size_records
         self.clock_resolution_usecs = clock_resolution_usecs
         self.provisioning_timeout_secs = provisioning_timeout_secs
-        self.runtime_version = runtime_version or os.environ.get(
-            "FELDERA_RUNTIME_VERSION"
-        )
         if fault_tolerance_model is not None:
             self.fault_tolerance = {
                 "model": str(fault_tolerance_model),
@@ -101,10 +96,13 @@ class RuntimeConfig:
             }
         if resources is not None:
             self.resources = resources.__dict__
-        if isinstance(storage, bool):
-            self.storage = storage
-        if isinstance(storage, Storage):
-            self.storage = storage.__dict__
+        if storage is not None:
+            if isinstance(storage, bool):
+                self.storage = storage
+            elif isinstance(storage, Storage):
+                self.storage = storage.__dict__
+            else:
+                raise ValueError(f"Unknown value '{storage}' for storage")
 
     @staticmethod
     def default() -> "RuntimeConfig":
@@ -119,3 +117,6 @@ class RuntimeConfig:
         conf = cls()
         conf.__dict__ = d
         return conf
+
+    def to_dict(self) -> dict:
+        return dict((k, v) for k, v in self.__dict__.items() if v is not None)

{feldera-0.108.0.dist-info → feldera-0.110.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.108.0
+Version: 0.110.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

{feldera-0.108.0.dist-info → feldera-0.110.0.dist-info}/RECORD

@@ -3,21 +3,21 @@ feldera/_callback_runner.py,sha256=GNOg3TrKJg9zJU0HvpWxCHqzjMUX8ORiHhtiEEdVQzE,4
 feldera/_helpers.py,sha256=rN0WuGSCCQlXWFMimZUQrgs-LJAfUo074d79sLElncQ,3023
 feldera/enums.py,sha256=MTHBojVANsdRnjbrzCyIOniDIUaH8nTYRfxB7QvajEE,9570
 feldera/output_handler.py,sha256=64J3ljhOaKIhxdjOKYi-BUz_HnMwROfmN8eE-btYygU,1930
-feldera/pipeline.py,sha256=6v87x_knsTlCEkflc5M-toSpHdhNGAfKQROjyY5neWk,34451
-feldera/pipeline_builder.py,sha256=pK8t93PIyM3zHzVTNg2jAafI7g2ZThNQyZER-_6e3G0,4081
-feldera/runtime_config.py,sha256=rPO1abLSJmozR4z1QwI93kt1rZMkFq-CBdqqmpKCSMU,4369
+feldera/pipeline.py,sha256=fRtGWYzxZOm6Z93bDjpU68EZ_uDTMdwPL-r-o7EXPTE,35135
+feldera/pipeline_builder.py,sha256=25tncJd-qiuHWZOezU34oGfDJqFAdjBEMd9QipNfswc,4195
+feldera/runtime_config.py,sha256=MuYJPd5G_hnu_eDz4ge4BfYvSBSOvOEtv4NYh5sEwqU,4452
 feldera/stats.py,sha256=XBhkRsV7FXErwWuPP0i3q9W77mzkMo-oThPVEZy5y3U,5028
 feldera/rest/__init__.py,sha256=Eg-EKUU3RSTDcdxTR_7wNDnCly8VpXEzsZCQUmf-y2M,308
 feldera/rest/_helpers.py,sha256=q7jWInKp9IiIli8N5o31lDG3hNUbcsJqufZXYHG04ps,222
 feldera/rest/_httprequests.py,sha256=e22YbpzOzy7MGo7hk9MOU7ZRTj3F314grY0Ygr-_goI,6636
 feldera/rest/config.py,sha256=DYzZKngDEhouTEwqVFd-rDrBN9tWqsU07Jl_BTT4mXs,1008
 feldera/rest/errors.py,sha256=b4i2JjrbSmej7jdko_FL8UeXklLKenSipwMT80jowaM,1720
-feldera/rest/feldera_client.py,sha256=Aic8PvBMG_aP_7d1qKx2C3RZ44rwLXN-OjvtV-a_dE8,25275
+feldera/rest/feldera_client.py,sha256=b85YS7gbxpCu5OHK34TDlB96vnt-HYVXY65WeWroDJY,25737
 feldera/rest/feldera_config.py,sha256=1pnGbLFMSLvp7Qh_OlPLALSKCSHIktNWKvx6gYU00U4,1374
 feldera/rest/pipeline.py,sha256=-dGGUdtHMABKrQEclaeuwGI_FOCrQOk6p2aCFV0FdU8,2890
 feldera/rest/sql_table.py,sha256=qrw-YwMzx5T81zDefNO1KOx7EyypFz1vPwGBzSUB7kc,652
 feldera/rest/sql_view.py,sha256=hN12mPM0mvwLCIPYywpb12s9Hd2Ws31IlTMXPriMisw,644
-feldera-0.108.0.dist-info/METADATA,sha256=ajNSOM-cPjqNnnpInK334SvP38Igrh4ychrz-zDfkWg,4101
-feldera-0.108.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-feldera-0.108.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
-feldera-0.108.0.dist-info/RECORD,,
+feldera-0.110.0.dist-info/METADATA,sha256=N93J-gZZsEyKYrckWeOEkhSQQYKQPc5xta5ltiFdCwk,4101
+feldera-0.110.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+feldera-0.110.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
+feldera-0.110.0.dist-info/RECORD,,