feldera 0.27.0__py3-none-any.whl

feldera/rest/errors.py

@@ -0,0 +1,55 @@
+from requests import Response
+import json
+
+
+class FelderaError(Exception):
+    """
+    Generic class for Feldera error handling
+    """
+
+    def __init__(self, message: str) -> None:
+        self.message = message
+        super().__init__(self.message)
+
+    def __str__(self) -> str:
+        return f"FelderaError. Error message: {self.message}"
+
+
+class FelderaAPIError(FelderaError):
+    """Error sent by Feldera API"""
+
+    def __init__(self, error: str, request: Response) -> None:
+        self.status_code = request.status_code
+        self.error = error
+        self.error_code = None
+        self.message = None
+        self.details = None
+
+        if request.text:
+            try:
+                json_data = json.loads(request.text)
+                self.message = json_data.get("message")
+                self.details = json_data.get("details")
+                self.error_code = json_data.get("error_code")
+            except:
+                self.message = request.text
+
+    def __str__(self) -> str:
+        if self.error_code:
+            return f"FelderaAPIError: {self.error}\n Error code: {self.error_code}\n Error message: {self.message}\n Details: {self.details}"
+        else:
+            return f"FelderaAPIError: {self.error}\n {self.message}"
+
+
+class FelderaTimeoutError(FelderaError):
+    """Error when Feldera operation takes longer than expected"""
+
+    def __str__(self) -> str:
+        return f"FelderaTimeoutError: {self.message}"
+
+
+class FelderaCommunicationError(FelderaError):
+    """Error when connection to Feldera"""
+
+    def __str__(self) -> str:
+        return f"FelderaCommunicationError: {self.message}"

feldera/rest/feldera_client.py

@@ -0,0 +1,377 @@
+from typing import Optional
+import logging
+import time
+import json
+from decimal import Decimal
+
+from feldera.rest.config import Config
+from feldera.rest.pipeline import Pipeline
+from feldera.rest._httprequests import HttpRequests
+
+
+def _prepare_boolean_input(value: bool) -> str:
+    return "true" if value else "false"
+
+
+class FelderaClient:
+    """
+    A client for the Feldera HTTP API
+
+    A client instance is needed for every Feldera API method to know the location of
+    Feldera and its permissions.
+    """
+
+    def __init__(
+            self,
+            url: str,
+            api_key: Optional[str] = None,
+            timeout: Optional[int] = None,
+    ) -> None:
+        """
+        :param url: The url to Feldera API (ex: https://try.feldera.com)
+        :param api_key: The optional API key for Feldera
+        :param timeout: (optional) The amount of time in seconds that the client will wait for a response before timing
+            out.
+        """
+
+        self.config = Config(url, api_key, timeout)
+        self.http = HttpRequests(self.config)
+
+        try:
+            self.pipelines()
+        except Exception as e:
+            logging.error(f"Failed to connect to Feldera API: {e}")
+            raise e
+
+    def get_pipeline(self, pipeline_name) -> Pipeline:
+        """
+        Get a pipeline by name
+
+        :param pipeline_name: The name of the pipeline
+        """
+
+        resp = self.http.get(f"/pipelines/{pipeline_name}")
+
+        return Pipeline.from_dict(resp)
+
+    def get_runtime_config(self, pipeline_name) -> dict:
+        """
+        Get the runtime config of a pipeline by name
+
+        :param pipeline_name: The name of the pipeline
+        """
+
+        resp: dict = self.http.get(f"/pipelines/{pipeline_name}")
+
+        return resp.get("runtime_config")
+
+    def pipelines(self) -> list[Pipeline]:
+        """
+        Get all pipelines
+        """
+
+        resp = self.http.get(
+            path="/pipelines",
+        )
+
+        return [Pipeline.from_dict(pipeline) for pipeline in resp]
+
+    def __wait_for_compilation(self, name: str):
+        wait = ["Pending", "CompilingSql", "CompilingRust"]
+
+        while True:
+            p = self.get_pipeline(name)
+            status = p.program_status
+
+            if status == "Success":
+                return p
+            elif status not in wait:
+                # TODO: return a more detailed error message
+                raise RuntimeError(f"The program failed to compile: {status}")
+
+            logging.debug("still compiling %s, waiting for 100 more milliseconds", name)
+            time.sleep(0.1)
+
+    def create_pipeline(self, pipeline: Pipeline) -> Pipeline:
+        """
+        Create a pipeline if it doesn't exist and wait for it to compile
+
+
+        :name: The name of the pipeline
+        """
+
+        body = {
+            "name": pipeline.name,
+            "program_code": pipeline.program_code,
+            "program_config": pipeline.program_config,
+            "runtime_config": pipeline.runtime_config,
+            "description": pipeline.description or "",
+        }
+
+        self.http.post(
+            path=f"/pipelines",
+            body=body,
+        )
+
+        return self.__wait_for_compilation(pipeline.name)
+
+    def create_or_update_pipeline(self, pipeline: Pipeline) -> Pipeline:
+        """
+        Create a pipeline if it doesn't exist or update a pipeline and wait for it to compile
+        """
+
+        body = {
+            "name": pipeline.name,
+            "program_code": pipeline.program_code,
+            "program_config": pipeline.program_config,
+            "runtime_config": pipeline.runtime_config,
+            "description": pipeline.description or "",
+        }
+
+        self.http.put(
+            path=f"/pipelines/{pipeline.name}",
+            body=body,
+        )
+
+        return self.__wait_for_compilation(pipeline.name)
+
+    def patch_pipeline(self, name: str, sql: str):
+        """
+        Incrementally update the pipeline SQL
+
+        :param name: The name of the pipeline
+        :param sql: The SQL snippet. Replaces the existing SQL code with this one.
+        """
+
+        self.http.patch(
+            path=f"/pipelines/{name}",
+            body={"program_code": sql},
+        )
+
+    def delete_pipeline(self, name: str):
+        """
+        Deletes a pipeline by name
+
+        :param name: The name of the pipeline
+        """
+        resp = self.http.delete(
+            path=f"/pipelines/{name}",
+        )
+
+    def get_pipeline_stats(self, name: str) -> dict:
+        """
+        Get the pipeline metrics and performance counters
+
+        :param name: The name of the pipeline
+        """
+
+        resp = self.http.get(
+            path=f"/pipelines/{name}/stats",
+        )
+
+        return resp
+
+    def start_pipeline(self, pipeline_name: str):
+        """
+        Start a pipeline
+
+        :param pipeline_name: The name of the pipeline to start
+        """
+
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/start",
+        )
+
+        while True:
+            status = self.get_pipeline(pipeline_name).deployment_status
+
+            if status == "Running":
+                break
+            elif status == "Failed":
+                raise RuntimeError(f"Failed to start pipeline")
+
+            logging.debug("still starting %s, waiting for 100 more milliseconds", pipeline_name)
+            time.sleep(0.1)
+
+    def pause_pipeline(self, pipeline_name: str):
+        """
+        Stop a pipeline
+
+        :param pipeline_name: The name of the pipeline to stop
+        """
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/pause",
+        )
+
+        while True:
+            status = self.get_pipeline(pipeline_name).deployment_status
+
+            if status == "Paused":
+                break
+            elif status == "Failed":
+                # TODO: return a more detailed error message
+                raise RuntimeError(f"Failed to pause pipeline")
+
+            logging.debug("still pausing %s, waiting for 100 more milliseconds", pipeline_name)
+            time.sleep(0.1)
+
+    def shutdown_pipeline(self, pipeline_name: str):
+        """
+        Shutdown a pipeline
+
+        :param pipeline_name: The name of the pipeline to shutdown
+        """
+
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/shutdown",
+        )
+
+        start = time.time()
+        timeout = 15
+
+        while time.time() - start < timeout:
+            status = self.get_pipeline(pipeline_name).deployment_status
+
+            if status == "Shutdown":
+                return
+
+            logging.debug("still shutting down %s, waiting for 100 more milliseconds", pipeline_name)
+            time.sleep(0.1)
+
+        # retry sending shutdown request as the pipline hasn't shutdown yet
+        logging.debug("pipeline %s hasn't shutdown after %s s, retrying", pipeline_name, timeout)
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/shutdown",
+        )
+
+        start = time.time()
+        timeout = 5
+
+        while time.time() - start < timeout:
+            status = self.get_pipeline(pipeline_name).deployment_status
+
+            if status == "Shutdown":
+                return
+
+            logging.debug("still shutting down %s, waiting for 100 more milliseconds", pipeline_name)
+            time.sleep(0.1)
+
+        raise RuntimeError(f"Failed to shutdown pipeline {pipeline_name}")
+
+    def push_to_pipeline(
+            self,
+            pipeline_name: str,
+            table_name: str,
+            format: str,
+            data: list[list | str | dict],
+            array: bool = False,
+            force: bool = False,
+            update_format: str = "raw",
+            json_flavor: str = None,
+            serialize: bool = True,
+    ):
+        """
+        Insert data into a pipeline
+
+        :param pipeline_name: The name of the pipeline
+        :param table_name: The name of the table
+        :param format: The format of the data, either "json" or "csv"
+        :param array: True if updates in this stream are packed into JSON arrays, used in conjunction with the
+            "json" format
+
+        :param force: If True, the data will be inserted even if the pipeline is paused
+        :param update_format: JSON data change event format, used in conjunction with the "json" format,
+            the default value is "insert_delete", other supported formats: "weighted", "debezium", "snowflake", "raw"
+        :param json_flavor: JSON encoding used for individual table records, the default value is "default", other supported encodings:
+            "debezium_mysql", "snowflake", "kafka_connect_json_converter", "pandas"
+        :param data: The data to insert
+        :param serialize: If True, the data will be serialized to JSON. True by default
+        """
+
+        if format not in ["json", "csv"]:
+            raise ValueError("format must be either 'json' or 'csv'")
+
+        if update_format not in ["insert_delete", "weighted", "debezium", "snowflake", "raw"]:
+            raise ValueError("update_format must be one of 'insert_delete', 'weighted', 'debezium', 'snowflake', 'raw'")
+
+        if json_flavor is not None and json_flavor not in ["default", "debezium_mysql", "snowflake", "kafka_connect_json_converter", "pandas"]:
+            raise ValueError("json_flavor must be one of 'default', 'debezium_mysql', 'snowflake', 'kafka_connect_json_converter', 'pandas'")
+
+        # python sends `True` which isn't accepted by the backend
+        array = _prepare_boolean_input(array)
+        force = _prepare_boolean_input(force)
+
+        params = {
+            "force": force,
+            "format": format,
+        }
+
+        if format == "json":
+            params["array"] = array
+            params["update_format"] = update_format
+
+        if json_flavor is not None:
+            params["json_flavor"] = json_flavor
+
+        content_type = "application/json"
+
+        if format == "csv":
+            content_type = "text/csv"
+            data = bytes(str(data), "utf-8")
+
+        self.http.post(
+            path=f"/pipelines/{pipeline_name}/ingress/{table_name}",
+            params=params,
+            content_type=content_type,
+            body=data,
+            serialize=serialize,
+        )
+
+    def listen_to_pipeline(
+            self,
+            pipeline_name: str,
+            table_name: str,
+            format: str,
+            backpressure: bool = True,
+            array: bool = False,
+            timeout: Optional[float] = None,
+    ):
+        """
+        Listen for updates to views for pipeline, yields the chunks of data
+
+        :param pipeline_name: The name of the pipeline
+        :param table_name: The name of the table to listen to
+        :param format: The format of the data, either "json" or "csv"
+        :param backpressure: When the flag is True (the default), this method waits for the consumer to receive each
+            chunk and blocks the pipeline if the consumer cannot keep up. When this flag is False, the pipeline drops
+            data chunks if the consumer is not keeping up with its output. This prevents a slow consumer from slowing
+            down the entire pipeline.
+        :param array: Set True to group updates in this stream into JSON arrays, used in conjunction with the
+            "json" format, the default value is False
+
+        :param timeout: The amount of time in seconds to listen to the stream for
+        """
+
+        params = {
+            "format": format,
+            "backpressure": _prepare_boolean_input(backpressure),
+        }
+
+        if format == "json":
+            params["array"] = _prepare_boolean_input(array)
+
+        resp = self.http.post(
+            path=f"/pipelines/{pipeline_name}/egress/{table_name}",
+            params=params,
+            stream=True,
+        )
+
+        end = time.time() + timeout if timeout else None
+
+        # Using the default chunk size below makes `iter_lines` extremely
+        # inefficient when dealing with long lines.
+        for chunk in resp.iter_lines(chunk_size=50000000):
+            if end and time.time() > end:
+                break
+            if chunk:
+                yield json.loads(chunk, parse_float=Decimal)

feldera/rest/pipeline.py

@@ -0,0 +1,69 @@
+from typing import Mapping, Any, Optional
+from feldera.rest.sql_table import SQLTable
+from feldera.rest.sql_view import SQLView
+
+
+class Pipeline:
+    """
+    Represents a Feldera pipeline
+    """
+
+    def __init__(
+            self,
+            name: str,
+            sql: str,
+            program_config: Mapping[str, Any],
+            runtime_config: Mapping[str, Any],
+            description: Optional[str] = None,
+    ):
+        """
+        Initializes a new pipeline
+
+        :param name: The name of the pipeline
+        :param sql: The SQL code of the pipeline
+        :param program_config: The program config of the pipeline
+        :param runtime_config: The configuration of the pipeline
+        :param description: Optional. The description of the pipeline
+        """
+
+        self.name: str = name
+        self.program_code: str = sql.strip()
+        self.description: Optional[str] = description
+        self.program_config: Mapping[str, Any] = program_config
+        self.runtime_config: Mapping[str, Any] = runtime_config
+        self.id: Optional[str] = id
+        self.tables: list[SQLTable] = []
+        self.views: list[SQLView] = []
+        self.deployment_status: Optional[str] = None
+        self.deployment_status_since: Optional[str] = None
+        self.created_at: Optional[str] = None
+        self.version: Optional[int] = None
+        self.program_version: Optional[int] = None
+        self.deployment_config: Optional[dict] = None
+        self.deployment_desired_status: Optional[str] = None
+        self.deployment_error: Optional[dict] = None
+        self.deployment_location: Optional[str] = None
+        self.program_binary_url: Optional[str] = None
+        self.program_info: Optional[dict] = None  # info about input & output connectors and the schema
+        self.program_status: Optional[str] = None
+        self.program_status_since: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, d: Mapping[str, Any]):
+        pipeline = cls("", "", {}, {})
+        pipeline.__dict__ = d
+        pipeline.tables = []
+        pipeline.views = []
+
+        info = d.get("program_info")
+
+        if info is not None:
+            for i in info['schema']['inputs']:
+                tbl = SQLTable.from_dict(i)
+                pipeline.tables.append(tbl)
+
+            for output in info['schema']['outputs']:
+                v = SQLView.from_dict(output)
+                pipeline.views.append(v)
+
+        return pipeline

feldera/rest/sql_table.py

@@ -0,0 +1,17 @@
+class SQLTable:
+    """
+    Represents a SQL table in Feldera
+    """
+
+    def __init__(self, name: str, fields: list[dict], case_sensitive: bool = False, materialized: bool = False):
+        self.name = name
+        self.case_sensitive = case_sensitive
+        self.materialized = materialized
+        self.fields: list[dict] = fields
+
+    @classmethod
+    def from_dict(self, table_dict: dict):
+        tbl = SQLTable(name=table_dict['name'], fields=table_dict['fields'])
+        tbl.case_sensitive = table_dict['case_sensitive']
+        tbl.materialized = table_dict['materialized']
+        return tbl

feldera/rest/sql_view.py

@@ -0,0 +1,17 @@
+class SQLView:
+    """
+    Represents a SQL view in Feldera
+    """
+
+    def __init__(self, name: str, fields: list[dict], case_sensitive: bool = False, materialized: bool = False):
+        self.name = name
+        self.case_sensitive = case_sensitive
+        self.materialized = materialized
+        self.fields: list[dict] = fields
+
+    @classmethod
+    def from_dict(self, view_dict: dict):
+        tbl = SQLView(name=view_dict['name'], fields=view_dict['fields'])
+        tbl.case_sensitive = view_dict['case_sensitive']
+        tbl.materialized = view_dict['materialized']
+        return tbl

feldera/runtime_config.py

@@ -0,0 +1,79 @@
+from typing import Optional, Any, Mapping
+
+
+class Resources:
+    """
+    Class used to specify the resource configuration for a pipeline.
+
+    :param config: A dictionary containing all the configuration values.
+    :param cpu_cores_max: The maximum number of CPU cores to reserve for an instance of the pipeline.
+    :param cpu_cores_min: The minimum number of CPU cores to reserve for an instance of the pipeline.
+    :param memory_mb_max: The maximum memory in Megabytes to reserve for an instance of the pipeline.
+    :param memory_mb_min: The minimum memory in Megabytes to reserve for an instance of the pipeline.
+    :param storage_class: The storage class to use for the pipeline. The class determines storage performance such
+        as IOPS and throughput.
+    :param storage_mb_max: The  storage in Megabytes to reserve for an instance of the pipeline.
+    """
+
+    def __init__(
+        self,
+        config: Optional[Mapping[str, Any]] = None,
+        cpu_cores_max: Optional[int] = None,
+        cpu_cores_min: Optional[int] = None,
+        memory_mb_max: Optional[int] = None,
+        memory_mb_min: Optional[int] = None,
+        storage_class: Optional[str] = None,
+        storage_mb_max: Optional[int] = None,
+    ):
+
+        config = config or {}
+
+        self.cpu_cores_max = cpu_cores_max
+        self.cpu_cores_min = cpu_cores_min
+        self.memory_mb_max = memory_mb_max
+        self.memory_mb_min = memory_mb_min
+        self.storage_class = storage_class
+        self.storage_mb_max = storage_mb_max
+
+        self.__dict__.update(config)
+
+
+class RuntimeConfig:
+    """
+    Runtime configuration class to define the configuration for a pipeline.
+    """
+
+    def __init__(
+            self,
+            workers: Optional[int] = None,
+            storage: Optional[bool] = False,
+            tracing: Optional[bool] = False,
+            tracing_endpoint_jaeger: Optional[str] = "",
+            cpu_profiler: bool = True,
+            max_buffering_delay_usecs: int = 0,
+            min_batch_size_records: int = 0,
+            min_storage_bytes: Optional[int] = None,
+            clock_resolution_usecs: Optional[int] = None,
+            resources: Optional[Resources] = None,
+    ):
+        self.workers = workers
+        self.storage = storage
+        self.tracing = tracing
+        self.tracing_endpoint_jaeger = tracing_endpoint_jaeger
+        self.cpu_profiler = cpu_profiler
+        self.max_buffering_delay_usecs = max_buffering_delay_usecs
+        self.min_batch_size_records = min_batch_size_records
+        self.min_storage_bytes = min_storage_bytes
+        self.clock_resolution_usecs = clock_resolution_usecs
+        if resources is not None:
+            self.resources = resources.__dict__
+
+    @classmethod
+    def from_dict(cls, d: Mapping[str, Any]):
+        """
+        Create a `.RuntimeConfig` object from a dictionary.
+        """
+
+        conf = cls()
+        conf.__dict__ = d
+        return conf