feldera 0.34.1__py3-none-any.whl

feldera/pipeline_builder.py

@@ -0,0 +1,109 @@
+from feldera.rest.feldera_client import FelderaClient
+from feldera.rest.pipeline import Pipeline as InnerPipeline
+from feldera.pipeline import Pipeline
+from feldera.enums import CompilationProfile
+from feldera.runtime_config import RuntimeConfig, Resources
+from feldera.rest.errors import FelderaAPIError
+
+
+class PipelineBuilder:
+    """
+    A builder for creating a Feldera Pipeline.
+
+    :param client: The `.FelderaClient` instance
+    :param name: The name of the pipeline
+    :param description: The description of the pipeline
+    :param sql: The SQL code of the pipeline
+    :param udf_rust: Rust code for UDFs
+    :param udf_toml: Rust dependencies required by UDFs (in the TOML format)
+    :param compilation_profile: The compilation profile to use
+    :param runtime_config: The runtime config to use
+    """
+
+    def __init__(
+        self,
+        client: FelderaClient,
+        name: str,
+        sql: str,
+        udf_rust: str = "",
+        udf_toml: str = "",
+        description: str = "",
+        compilation_profile: CompilationProfile = CompilationProfile.OPTIMIZED,
+        runtime_config: RuntimeConfig = RuntimeConfig(resources=Resources()),
+    ):
+        self.client: FelderaClient = client
+        self.name: str | None = name
+        self.description: str = description
+        self.sql: str = sql
+        self.udf_rust: str = udf_rust
+        self.udf_toml: str = udf_toml
+        self.compilation_profile: CompilationProfile = compilation_profile
+        self.runtime_config: RuntimeConfig = runtime_config
+
+    def create(self) -> Pipeline:
+        """
+        Create the pipeline if it does not exist.
+
+        :return: The created pipeline
+        """
+
+        if self.name is None or self.sql is None:
+            raise ValueError("Name and SQL are required to create a pipeline")
+
+        if self.client.get_pipeline(self.name) is not None:
+            raise RuntimeError(f"Pipeline with name {self.name} already exists")
+
+        inner = InnerPipeline(
+            self.name,
+            description=self.description,
+            sql=self.sql,
+            udf_rust=self.udf_rust,
+            udf_toml=self.udf_toml,
+            program_config={
+                "profile": self.compilation_profile.value,
+            },
+            runtime_config=self.runtime_config.__dict__,
+        )
+
+        inner = self.client.create_pipeline(inner)
+        pipeline = Pipeline(self.client)
+        pipeline._inner = inner
+
+        return pipeline
+
+    def create_or_replace(self) -> Pipeline:
+        """
+        Creates a pipeline if it does not exist and replaces it if it exists.
+
+        If the pipeline exists and is running, it will be stopped and replaced.
+        """
+
+        if self.name is None or self.sql is None:
+            raise ValueError("Name and SQL are required to create a pipeline")
+
+        try:
+            # shutdown the pipeline if it exists and is running
+            self.client.shutdown_pipeline(self.name)
+        except FelderaAPIError:
+            # pipeline doesn't exist, no worries
+            pass
+
+        inner = InnerPipeline(
+            self.name,
+            description=self.description,
+            sql=self.sql,
+            udf_rust=self.udf_rust,
+            udf_toml=self.udf_toml,
+            program_config={
+                "profile": self.compilation_profile.value,
+            },
+            runtime_config=dict(
+                (k, v) for k, v in self.runtime_config.__dict__.items() if v is not None
+            ),
+        )
+
+        inner = self.client.create_or_update_pipeline(inner)
+        pipeline = Pipeline(self.client)
+        pipeline._inner = inner
+
+        return pipeline

feldera/rest/__init__.py

@@ -0,0 +1,11 @@
+"""
+This is the lower level REST client for Feldera.
+
+This is a thin wrapper around the Feldera REST API.
+
+It is recommended to use the higher level abstractions in the `feldera` package,
+instead of using the REST client directly.
+
+"""
+
+from feldera.rest.feldera_client import FelderaClient as FelderaClient

feldera/rest/_httprequests.py

@@ -0,0 +1,182 @@
+import logging
+
+from feldera.rest.config import Config
+
+from feldera.rest.errors import (
+    FelderaAPIError,
+    FelderaTimeoutError,
+    FelderaCommunicationError,
+)
+
+import json
+import requests
+from typing import Callable, Optional, Any, Union, Mapping, Sequence, List
+
+
+def json_serialize(body: Any) -> str:
+    return json.dumps(body) if body else "" if body == "" else "null"
+
+
+class HttpRequests:
+    def __init__(self, config: Config) -> None:
+        self.config = config
+        self.headers = {"User-Agent": "feldera-python-sdk/v1"}
+        if self.config.api_key:
+            self.headers["Authorization"] = f"Bearer {self.config.api_key}"
+
+    def send_request(
+        self,
+        http_method: Callable,
+        path: str,
+        body: Optional[
+            Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str], str]
+        ] = None,
+        content_type: str = "application/json",
+        params: Optional[Mapping[str, Any]] = None,
+        stream: bool = False,
+        serialize: bool = True,
+    ) -> Any:
+        """
+        :param http_method: The HTTP method to use. Takes the equivalent `requests.*` module. (Example: `requests.get`)
+        :param path: The path to send the request to.
+        :param body: The HTTP request body.
+        :param content_type: The value for `Content-Type` HTTP header. "application/json" by default.
+        :param params: The query parameters part of this request.
+        :param stream: True if the response is expected to be a HTTP stream.
+        :param serialize: True if the body needs to be serialized to JSON.
+        """
+        self.headers["Content-Type"] = content_type
+
+        try:
+            timeout = self.config.timeout
+            headers = self.headers
+
+            request_path = self.config.url + "/" + self.config.version + path
+
+            logging.debug(
+                "sending %s request to: %s with headers: %s, and params: %s",
+                http_method.__name__,
+                request_path,
+                str(headers),
+                str(params),
+            )
+
+            if http_method.__name__ == "get":
+                request = http_method(
+                    request_path,
+                    timeout=timeout,
+                    headers=headers,
+                    params=params,
+                    stream=stream,
+                )
+            elif isinstance(body, bytes):
+                request = http_method(
+                    request_path,
+                    timeout=timeout,
+                    headers=headers,
+                    data=body,
+                    params=params,
+                    stream=stream,
+                )
+            else:
+                request = http_method(
+                    request_path,
+                    timeout=timeout,
+                    headers=headers,
+                    data=json_serialize(body) if serialize else body,
+                    params=params,
+                    stream=stream,
+                )
+
+            resp = self.__validate(request, stream=stream)
+            logging.debug("got response: %s", str(resp))
+            return resp
+
+        except requests.exceptions.Timeout as err:
+            raise FelderaTimeoutError(str(err)) from err
+        except requests.exceptions.ConnectionError as err:
+            raise FelderaCommunicationError(str(err)) from err
+
+    def get(
+        self,
+        path: str,
+        params: Optional[Mapping[str, Any]] = None,
+        stream: bool = False,
+    ) -> Any:
+        return self.send_request(requests.get, path, params=params, stream=stream)
+
+    def post(
+        self,
+        path: str,
+        body: Optional[
+            Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str], str]
+        ] = None,
+        content_type: Optional[str] = "application/json",
+        params: Optional[Mapping[str, Any]] = None,
+        stream: bool = False,
+        serialize: bool = True,
+    ) -> Any:
+        return self.send_request(
+            requests.post,
+            path,
+            body,
+            content_type,
+            params,
+            stream=stream,
+            serialize=serialize,
+        )
+
+    def patch(
+        self,
+        path: str,
+        body: Optional[
+            Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str], str]
+        ] = None,
+        content_type: Optional[str] = "application/json",
+        params: Optional[Mapping[str, Any]] = None,
+    ) -> Any:
+        return self.send_request(requests.patch, path, body, content_type, params)
+
+    def put(
+        self,
+        path: str,
+        body: Optional[
+            Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str], str]
+        ] = None,
+        content_type: Optional[str] = "application/json",
+        params: Optional[Mapping[str, Any]] = None,
+    ) -> Any:
+        return self.send_request(requests.put, path, body, content_type, params)
+
+    def delete(
+        self,
+        path: str,
+        body: Optional[
+            Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str]]
+        ] = None,
+        params: Optional[Mapping[str, Any]] = None,
+    ) -> Any:
+        return self.send_request(requests.delete, path, body, params=params)
+
+    @staticmethod
+    def __to_json(request: requests.Response) -> Any:
+        if request.content == b"":
+            return request
+        return request.json()
+
+    @staticmethod
+    def __validate(request: requests.Response, stream=False) -> Any:
+        try:
+            request.raise_for_status()
+
+            if stream:
+                return request
+            if request.headers.get("content-type") == "text/plain":
+                return request.text
+            elif request.headers.get("content-type") == "application/octet-stream":
+                return request.content
+
+            resp = HttpRequests.__to_json(request)
+            return resp
+        except requests.exceptions.HTTPError as err:
+            raise FelderaAPIError(str(err), request) from err

feldera/rest/config.py

@@ -0,0 +1,26 @@
+from typing import Optional
+
+
+class Config:
+    """
+    :class:`.FelderaClient`'s credentials and configuration parameters
+    """
+
+    def __init__(
+        self,
+        url: str,
+        api_key: Optional[str] = None,
+        version: Optional[str] = None,
+        timeout: Optional[float] = None,
+    ) -> None:
+        """
+        :param url: The url to the Feldera API (ex: https://try.feldera.com)
+        :param api_key: The optional API key to access Feldera
+        :param version: The version of the API to use
+        :param timeout: The timeout for the HTTP requests
+        """
+
+        self.url: str = url
+        self.api_key: Optional[str] = api_key
+        self.version: Optional[str] = version or "v0"
+        self.timeout: Optional[float] = timeout

feldera/rest/errors.py

@@ -0,0 +1,58 @@
+from requests import Response
+import json
+
+
+class FelderaError(Exception):
+    """
+    Generic class for Feldera error handling
+    """
+
+    def __init__(self, message: str) -> None:
+        self.message = message
+        super().__init__(self.message)
+
+    def __str__(self) -> str:
+        return f"FelderaError. Error message: {self.message}"
+
+
+class FelderaAPIError(FelderaError):
+    """Error sent by Feldera API"""
+
+    def __init__(self, error: str, request: Response) -> None:
+        self.status_code = request.status_code
+        self.error = error
+        self.error_code = None
+        self.message = None
+        self.details = None
+
+        err_msg = ""
+
+        if request.text:
+            try:
+                json_data = json.loads(request.text)
+
+                self.error_code = json_data.get("error_code")
+                if self.error_code:
+                    err_msg += f"\nError Code: {self.error_code}"
+                self.message = json_data.get("message")
+                if self.message:
+                    err_msg += f"\nMessage: {self.message}"
+                self.details = json_data.get("details")
+            except Exception:
+                self.message = request.text
+
+        super().__init__(err_msg)
+
+
+class FelderaTimeoutError(FelderaError):
+    """Error when Feldera operation takes longer than expected"""
+
+    def __init__(self, err: str) -> None:
+        super().__init__(f"Timeout connecting to Feldera: {err}")
+
+
+class FelderaCommunicationError(FelderaError):
+    """Error when connection to Feldera"""
+
+    def __init__(self, err: str) -> None:
+        super().__init__(f"Cannot connect to Feldera API: {err}")