feldera 0.69.0__py3-none-any.whl → 0.189.0__py3-none-any.whl

feldera/pipeline_builder.py

@@ -1,23 +1,28 @@
-from feldera.rest.feldera_client import FelderaClient
-from feldera.rest.pipeline import Pipeline as InnerPipeline
+import os
+from typing import Optional
+
+from feldera.enums import CompilationProfile, PipelineFieldSelector
 from feldera.pipeline import Pipeline
-from feldera.enums import CompilationProfile
-from feldera.runtime_config import RuntimeConfig, Resources
 from feldera.rest.errors import FelderaAPIError
+from feldera.rest.feldera_client import FelderaClient
+from feldera.rest.pipeline import Pipeline as InnerPipeline
+from feldera.runtime_config import RuntimeConfig
 
 
 class PipelineBuilder:
     """
     A builder for creating a Feldera Pipeline.
 
-    :param client: The `.FelderaClient` instance
+    :param client: The :class:`.FelderaClient` instance
     :param name: The name of the pipeline
     :param description: The description of the pipeline
     :param sql: The SQL code of the pipeline
     :param udf_rust: Rust code for UDFs
     :param udf_toml: Rust dependencies required by UDFs (in the TOML format)
-    :param compilation_profile: The compilation profile to use
-    :param runtime_config: The runtime config to use
+    :param compilation_profile: The :class:`.CompilationProfile` to use
+    :param runtime_config: The :class:`.RuntimeConfig` to use. Enables
+        configuring the runtime behavior of the pipeline such as:
+        fault tolerance, storage and :class:`.Resources`
     """
 
     def __init__(
@@ -29,7 +34,8 @@ class PipelineBuilder:
         udf_toml: str = "",
         description: str = "",
         compilation_profile: CompilationProfile = CompilationProfile.OPTIMIZED,
-        runtime_config: RuntimeConfig = RuntimeConfig(resources=Resources()),
+        runtime_config: RuntimeConfig = RuntimeConfig.default(),
+        runtime_version: Optional[str] = None,
     ):
         self.client: FelderaClient = client
         self.name: str | None = name
@@ -39,19 +45,30 @@ class PipelineBuilder:
         self.udf_toml: str = udf_toml
         self.compilation_profile: CompilationProfile = compilation_profile
         self.runtime_config: RuntimeConfig = runtime_config
+        self.runtime_version: Optional[str] = os.environ.get(
+            "FELDERA_RUNTIME_VERSION", runtime_version
+        )
 
-    def create(self) -> Pipeline:
+    def create(self, wait: bool = True) -> Pipeline:
         """
         Create the pipeline if it does not exist.
 
+        :param wait: Whether to wait for the pipeline to be created. True by default
         :return: The created pipeline
         """
 
         if self.name is None or self.sql is None:
             raise ValueError("Name and SQL are required to create a pipeline")
 
-        if self.client.get_pipeline(self.name) is not None:
-            raise RuntimeError(f"Pipeline with name {self.name} already exists")
+        try:
+            if (
+                self.client.get_pipeline(self.name, PipelineFieldSelector.STATUS)
+                is not None
+            ):
+                raise RuntimeError(f"Pipeline with name {self.name} already exists")
+        except FelderaAPIError as err:
+            if err.error_code != "UnknownPipelineName":
+                raise err
 
         inner = InnerPipeline(
             self.name,
@@ -61,21 +78,25 @@ class PipelineBuilder:
             udf_toml=self.udf_toml,
             program_config={
                 "profile": self.compilation_profile.value,
+                "runtime_version": self.runtime_version,
             },
-            runtime_config=self.runtime_config.__dict__,
+            runtime_config=self.runtime_config.to_dict(),
         )
 
-        inner = self.client.create_pipeline(inner)
+        inner = self.client.create_pipeline(inner, wait=wait)
         pipeline = Pipeline(self.client)
         pipeline._inner = inner
 
         return pipeline
 
-    def create_or_replace(self) -> Pipeline:
+    def create_or_replace(self, wait: bool = True) -> Pipeline:
         """
         Creates a pipeline if it does not exist and replaces it if it exists.
 
         If the pipeline exists and is running, it will be stopped and replaced.
+
+        :param wait: Whether to wait for the pipeline to be created. True by default
+        :return: The created pipeline
         """
 
         if self.name is None or self.sql is None:
@@ -83,7 +104,10 @@ class PipelineBuilder:
 
         try:
             # shutdown the pipeline if it exists and is running
-            self.client.shutdown_pipeline(self.name)
+            p = Pipeline.get(self.name, self.client)
+            p.stop(force=True)
+            p.clear_storage()
+
         except FelderaAPIError:
             # pipeline doesn't exist, no worries
             pass
@@ -96,13 +120,12 @@ class PipelineBuilder:
             udf_toml=self.udf_toml,
             program_config={
                 "profile": self.compilation_profile.value,
+                "runtime_version": self.runtime_version,
             },
-            runtime_config=dict(
-                (k, v) for k, v in self.runtime_config.__dict__.items() if v is not None
-            ),
+            runtime_config=self.runtime_config.to_dict(),
         )
 
-        inner = self.client.create_or_update_pipeline(inner)
+        inner = self.client.create_or_update_pipeline(inner, wait=wait)
         pipeline = Pipeline(self.client)
         pipeline._inner = inner
 

feldera/rest/_helpers.py

@@ -0,0 +1,40 @@
+import logging
+import os
+
+
+def determine_client_version() -> str:
+    from importlib.metadata import version, PackageNotFoundError
+
+    try:
+        version = version("feldera")
+    except PackageNotFoundError:
+        version = "unknown"
+
+    return version
+
+
+def requests_verify_from_env() -> str | bool:
+    env_feldera_tls_insecure = os.environ.get("FELDERA_TLS_INSECURE")
+    FELDERA_HTTPS_TLS_CERT = os.environ.get("FELDERA_HTTPS_TLS_CERT")
+
+    if env_feldera_tls_insecure is not None and FELDERA_HTTPS_TLS_CERT is not None:
+        logging.warning(
+            "environment variables FELDERA_HTTPS_TLS_CERT and "
+            + "FELDERA_TLS_INSECURE both are set."
+            + "\nFELDERA_HTTPS_TLS_CERT takes priority."
+        )
+
+    if env_feldera_tls_insecure is None:
+        feldera_tls_insecure = False
+    else:
+        feldera_tls_insecure = env_feldera_tls_insecure.strip().lower() in (
+            "1",
+            "true",
+            "yes",
+        )
+
+    requests_verify = not feldera_tls_insecure
+    if FELDERA_HTTPS_TLS_CERT is not None:
+        requests_verify = FELDERA_HTTPS_TLS_CERT
+
+    return requests_verify