feldera 0.27.0__py3-none-any.whl → 0.29.0__py3-none-any.whl

feldera/rest/_httprequests.py

@@ -2,7 +2,11 @@ import logging
 
 from feldera.rest.config import Config
 
-from feldera.rest.errors import FelderaAPIError, FelderaTimeoutError, FelderaCommunicationError
+from feldera.rest.errors import (
+    FelderaAPIError,
+    FelderaTimeoutError,
+    FelderaCommunicationError,
+)
 
 import json
 import requests
@@ -16,23 +20,21 @@ def json_serialize(body: Any) -> str:
 class HttpRequests:
     def __init__(self, config: Config) -> None:
         self.config = config
-        self.headers = {
-            "User-Agent": "feldera-python-sdk/v1"
-        }
+        self.headers = {"User-Agent": "feldera-python-sdk/v1"}
         if self.config.api_key:
             self.headers["Authorization"] = f"Bearer {self.config.api_key}"
 
     def send_request(
-            self,
-            http_method: Callable,
-            path: str,
-            body: Optional[
-                Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str], str]
-            ] = None,
-            content_type: str = "application/json",
-            params: Optional[Mapping[str, Any]] = None,
-            stream: bool = False,
-            serialize: bool = True,
+        self,
+        http_method: Callable,
+        path: str,
+        body: Optional[
+            Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str], str]
+        ] = None,
+        content_type: str = "application/json",
+        params: Optional[Mapping[str, Any]] = None,
+        stream: bool = False,
+        serialize: bool = True,
     ) -> Any:
         """
         :param http_method: The HTTP method to use. Takes the equivalent `requests.*` module. (Example: `requests.get`)
@@ -53,7 +55,10 @@ class HttpRequests:
 
             logging.debug(
                 "sending %s request to: %s with headers: %s, and params: %s",
-                http_method.__name__, request_path, str(headers), str(params)
+                http_method.__name__,
+                request_path,
+                str(headers),
+                str(params),
             )
 
             if http_method.__name__ == "get":
@@ -62,6 +67,7 @@ class HttpRequests:
                     timeout=timeout,
                     headers=headers,
                     params=params,
+                    stream=stream,
                 )
             elif isinstance(body, bytes):
                 request = http_method(
@@ -81,9 +87,8 @@ class HttpRequests:
                     params=params,
                     stream=stream,
                 )
-                if stream:
-                    return request
-            resp = self.__validate(request)
+
+            resp = self.__validate(request, stream=stream)
             logging.debug("got response: %s", str(resp))
             return resp
 
@@ -93,59 +98,63 @@ class HttpRequests:
             raise FelderaCommunicationError(str(err)) from err
 
     def get(
-            self,
-            path: str,
-            params: Optional[Mapping[str, Any]] = None
+        self,
+        path: str,
+        params: Optional[Mapping[str, Any]] = None,
+        stream: bool = False,
     ) -> Any:
-        return self.send_request(requests.get, path, params)
+        return self.send_request(requests.get, path, params=params, stream=stream)
 
     def post(
-            self,
-            path: str,
-            body: Optional[
-                Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str], str]
-            ] = None,
-            content_type: Optional[str] = "application/json",
-            params: Optional[Mapping[str, Any]] = None,
-            stream: bool = False,
-            serialize: bool = True,
+        self,
+        path: str,
+        body: Optional[
+            Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str], str]
+        ] = None,
+        content_type: Optional[str] = "application/json",
+        params: Optional[Mapping[str, Any]] = None,
+        stream: bool = False,
+        serialize: bool = True,
     ) -> Any:
         return self.send_request(
             requests.post,
             path,
             body,
             content_type,
-            params, stream=stream,
-            serialize=serialize
+            params,
+            stream=stream,
+            serialize=serialize,
         )
 
     def patch(
-            self,
-            path: str,
-            body: Optional[
-                Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str], str]
-            ] = None,
-            content_type: Optional[str] = "application/json",
-            params: Optional[Mapping[str, Any]] = None
+        self,
+        path: str,
+        body: Optional[
+            Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str], str]
+        ] = None,
+        content_type: Optional[str] = "application/json",
+        params: Optional[Mapping[str, Any]] = None,
     ) -> Any:
         return self.send_request(requests.patch, path, body, content_type, params)
 
     def put(
-            self,
-            path: str,
-            body: Optional[
-                Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str], str]
-            ] = None,
-            content_type: Optional[str] = "application/json",
-            params: Optional[Mapping[str, Any]] = None
+        self,
+        path: str,
+        body: Optional[
+            Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str], str]
+        ] = None,
+        content_type: Optional[str] = "application/json",
+        params: Optional[Mapping[str, Any]] = None,
     ) -> Any:
         return self.send_request(requests.put, path, body, content_type, params)
 
     def delete(
-            self,
-            path: str,
-            body: Optional[Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str]]] = None,
-            params: Optional[Mapping[str, Any]] = None
+        self,
+        path: str,
+        body: Optional[
+            Union[Mapping[str, Any], Sequence[Mapping[str, Any]], List[str]]
+        ] = None,
+        params: Optional[Mapping[str, Any]] = None,
     ) -> Any:
         return self.send_request(requests.delete, path, body, params=params)
 
@@ -156,9 +165,17 @@ class HttpRequests:
         return request.json()
 
     @staticmethod
-    def __validate(request: requests.Response) -> Any:
+    def __validate(request: requests.Response, stream=False) -> Any:
         try:
             request.raise_for_status()
+
+            if stream:
+                return request
+            if request.headers.get("content-type") == "text/plain":
+                return request.text
+            elif request.headers.get("content-type") == "application/octet-stream":
+                return request.content
+
             resp = HttpRequests.__to_json(request)
             return resp
         except requests.exceptions.HTTPError as err:

feldera/rest/config.py

@@ -7,11 +7,11 @@ class Config:
     """
 
     def __init__(
-            self,
-            url: str,
-            api_key: Optional[str] = None,
-            version: Optional[str] = None,
-            timeout: Optional[float] = None,
+        self,
+        url: str,
+        api_key: Optional[str] = None,
+        version: Optional[str] = None,
+        timeout: Optional[float] = None,
     ) -> None:
         """
         :param url: The url to the Feldera API (ex: https://try.feldera.com)

feldera/rest/errors.py

@@ -25,31 +25,34 @@ class FelderaAPIError(FelderaError):
         self.message = None
         self.details = None
 
+        err_msg = ""
+
         if request.text:
             try:
                 json_data = json.loads(request.text)
+
+                self.error_code = json_data.get("error_code")
+                if self.error_code:
+                    err_msg += f"\nError Code: {self.error_code}"
                 self.message = json_data.get("message")
+                if self.message:
+                    err_msg += f"\nMessage: {self.message}"
                 self.details = json_data.get("details")
-                self.error_code = json_data.get("error_code")
-            except:
+            except Exception:
                 self.message = request.text
 
-    def __str__(self) -> str:
-        if self.error_code:
-            return f"FelderaAPIError: {self.error}\n Error code: {self.error_code}\n Error message: {self.message}\n Details: {self.details}"
-        else:
-            return f"FelderaAPIError: {self.error}\n {self.message}"
+        super().__init__(err_msg)
 
 
 class FelderaTimeoutError(FelderaError):
     """Error when Feldera operation takes longer than expected"""
 
-    def __str__(self) -> str:
-        return f"FelderaTimeoutError: {self.message}"
+    def __init__(self, err: str) -> None:
+        super().__init__(f"Timeout connecting to Feldera: {err}")
 
 
 class FelderaCommunicationError(FelderaError):
     """Error when connection to Feldera"""
 
-    def __str__(self) -> str:
-        return f"FelderaCommunicationError: {self.message}"
+    def __init__(self, err: str) -> None:
+        super().__init__(f"Cannot connect to Feldera API: {err}")

feldera/rest/feldera_client.py

@@ -1,8 +1,10 @@
+import pathlib
 from typing import Optional
 import logging
 import time
 import json
 from decimal import Decimal
+from typing import Generator
 
 from feldera.rest.config import Config
 from feldera.rest.pipeline import Pipeline
@@ -22,10 +24,10 @@ class FelderaClient:
     """
 
     def __init__(
-            self,
-            url: str,
-            api_key: Optional[str] = None,
-            timeout: Optional[int] = None,
+        self,
+        url: str,
+        api_key: Optional[str] = None,
+        timeout: Optional[int] = None,
     ) -> None:
         """
         :param url: The url to Feldera API (ex: https://try.feldera.com)
@@ -86,7 +88,18 @@ class FelderaClient:
             if status == "Success":
                 return p
             elif status not in wait:
-                # TODO: return a more detailed error message
+                # error handling for SQL compilation errors
+                if isinstance(status, dict):
+                    sql_errors = status.get("SqlError")
+                    if sql_errors:
+                        err_msg = f"Pipeline {name} failed to compile:\n"
+                        for sql_error in sql_errors:
+                            err_msg += (
+                                f"{sql_error['error_type']}\n{sql_error['message']}\n"
+                            )
+                            err_msg += f"Code snippet:\n{sql_error['snippet']}"
+                        raise RuntimeError(err_msg)
+
                 raise RuntimeError(f"The program failed to compile: {status}")
 
             logging.debug("still compiling %s, waiting for 100 more milliseconds", name)
@@ -103,13 +116,15 @@ class FelderaClient:
         body = {
             "name": pipeline.name,
             "program_code": pipeline.program_code,
+            "udf_rust": pipeline.udf_rust,
+            "udf_toml": pipeline.udf_toml,
             "program_config": pipeline.program_config,
             "runtime_config": pipeline.runtime_config,
             "description": pipeline.description or "",
         }
 
         self.http.post(
-            path=f"/pipelines",
+            path="/pipelines",
             body=body,
         )
 
@@ -123,6 +138,8 @@ class FelderaClient:
         body = {
             "name": pipeline.name,
             "program_code": pipeline.program_code,
+            "udf_rust": pipeline.udf_rust,
+            "udf_toml": pipeline.udf_toml,
             "program_config": pipeline.program_config,
             "runtime_config": pipeline.runtime_config,
             "description": pipeline.description or "",
@@ -154,7 +171,7 @@ class FelderaClient:
 
         :param name: The name of the pipeline
         """
-        resp = self.http.delete(
+        self.http.delete(
             path=f"/pipelines/{name}",
         )
 
@@ -188,9 +205,11 @@ class FelderaClient:
             if status == "Running":
                 break
             elif status == "Failed":
-                raise RuntimeError(f"Failed to start pipeline")
+                raise RuntimeError("Failed to start pipeline")
 
-            logging.debug("still starting %s, waiting for 100 more milliseconds", pipeline_name)
+            logging.debug(
+                "still starting %s, waiting for 100 more milliseconds", pipeline_name
+            )
             time.sleep(0.1)
 
     def pause_pipeline(self, pipeline_name: str):
@@ -209,17 +228,18 @@ class FelderaClient:
             if status == "Paused":
                 break
             elif status == "Failed":
-                # TODO: return a more detailed error message
-                raise RuntimeError(f"Failed to pause pipeline")
+                raise RuntimeError("Failed to pause pipeline")
 
-            logging.debug("still pausing %s, waiting for 100 more milliseconds", pipeline_name)
+            logging.debug(
+                "still pausing %s, waiting for 100 more milliseconds", pipeline_name
+            )
             time.sleep(0.1)
 
     def shutdown_pipeline(self, pipeline_name: str):
         """
         Shutdown a pipeline
 
-        :param pipeline_name: The name of the pipeline to shutdown
+        :param pipeline_name: The name of the pipeline to shut down
         """
 
         self.http.post(
@@ -235,11 +255,16 @@ class FelderaClient:
             if status == "Shutdown":
                 return
 
-            logging.debug("still shutting down %s, waiting for 100 more milliseconds", pipeline_name)
+            logging.debug(
+                "still shutting down %s, waiting for 100 more milliseconds",
+                pipeline_name,
+            )
             time.sleep(0.1)
 
         # retry sending shutdown request as the pipline hasn't shutdown yet
-        logging.debug("pipeline %s hasn't shutdown after %s s, retrying", pipeline_name, timeout)
+        logging.debug(
+            "pipeline %s hasn't shutdown after %s s, retrying", pipeline_name, timeout
+        )
         self.http.post(
             path=f"/pipelines/{pipeline_name}/shutdown",
         )
@@ -253,22 +278,25 @@ class FelderaClient:
             if status == "Shutdown":
                 return
 
-            logging.debug("still shutting down %s, waiting for 100 more milliseconds", pipeline_name)
+            logging.debug(
+                "still shutting down %s, waiting for 100 more milliseconds",
+                pipeline_name,
+            )
             time.sleep(0.1)
 
         raise RuntimeError(f"Failed to shutdown pipeline {pipeline_name}")
 
     def push_to_pipeline(
-            self,
-            pipeline_name: str,
-            table_name: str,
-            format: str,
-            data: list[list | str | dict],
-            array: bool = False,
-            force: bool = False,
-            update_format: str = "raw",
-            json_flavor: str = None,
-            serialize: bool = True,
+        self,
+        pipeline_name: str,
+        table_name: str,
+        format: str,
+        data: list[list | str | dict],
+        array: bool = False,
+        force: bool = False,
+        update_format: str = "raw",
+        json_flavor: str = None,
+        serialize: bool = True,
     ):
         """
         Insert data into a pipeline
@@ -291,11 +319,27 @@ class FelderaClient:
         if format not in ["json", "csv"]:
             raise ValueError("format must be either 'json' or 'csv'")
 
-        if update_format not in ["insert_delete", "weighted", "debezium", "snowflake", "raw"]:
-            raise ValueError("update_format must be one of 'insert_delete', 'weighted', 'debezium', 'snowflake', 'raw'")
-
-        if json_flavor is not None and json_flavor not in ["default", "debezium_mysql", "snowflake", "kafka_connect_json_converter", "pandas"]:
-            raise ValueError("json_flavor must be one of 'default', 'debezium_mysql', 'snowflake', 'kafka_connect_json_converter', 'pandas'")
+        if update_format not in [
+            "insert_delete",
+            "weighted",
+            "debezium",
+            "snowflake",
+            "raw",
+        ]:
+            raise ValueError(
+                "update_format must be one of 'insert_delete', 'weighted', 'debezium', 'snowflake', 'raw'"
+            )
+
+        if json_flavor is not None and json_flavor not in [
+            "default",
+            "debezium_mysql",
+            "snowflake",
+            "kafka_connect_json_converter",
+            "pandas",
+        ]:
+            raise ValueError(
+                "json_flavor must be one of 'default', 'debezium_mysql', 'snowflake', 'kafka_connect_json_converter', 'pandas'"
+            )
 
         # python sends `True` which isn't accepted by the backend
         array = _prepare_boolean_input(array)
@@ -328,13 +372,13 @@ class FelderaClient:
         )
 
     def listen_to_pipeline(
-            self,
-            pipeline_name: str,
-            table_name: str,
-            format: str,
-            backpressure: bool = True,
-            array: bool = False,
-            timeout: Optional[float] = None,
+        self,
+        pipeline_name: str,
+        table_name: str,
+        format: str,
+        backpressure: bool = True,
+        array: bool = False,
+        timeout: Optional[float] = None,
     ):
         """
         Listen for updates to views for pipeline, yields the chunks of data
@@ -375,3 +419,93 @@ class FelderaClient:
                 break
             if chunk:
                 yield json.loads(chunk, parse_float=Decimal)
+
+    def query_as_text(
+        self, pipeline_name: str, query: str
+    ) -> Generator[str, None, None]:
+        """
+        Executes an ad-hoc query on the specified pipeline and returns a generator that yields lines of the table.
+
+        :param pipeline_name: The name of the pipeline to query.
+        :param query: The SQL query to be executed.
+        :return: A generator yielding the query result in tabular format, one line at a time.
+        """
+        params = {
+            "pipeline_name": pipeline_name,
+            "sql": query,
+            "format": "text",
+        }
+
+        resp = self.http.get(
+            path=f"/pipelines/{pipeline_name}/query",
+            params=params,
+            stream=True,
+        )
+
+        chunk: bytes
+        for chunk in resp.iter_lines(chunk_size=50000000):
+            if chunk:
+                yield chunk.decode("utf-8")
+
+    def query_as_parquet(self, pipeline_name: str, query: str, path: str):
+        """
+        Executes an ad-hoc query on the specified pipeline and saves the result to a parquet file.
+        If the extension isn't `parquet`, it will be automatically appended to `path`.
+
+        :param pipeline_name: The name of the pipeline to query.
+        :param query: The SQL query to be executed.
+        :param path: The path including the file name to save the resulting parquet file in.
+        """
+
+        params = {
+            "pipeline_name": pipeline_name,
+            "sql": query,
+            "format": "parquet",
+        }
+
+        resp = self.http.get(
+            path=f"/pipelines/{pipeline_name}/query",
+            params=params,
+            stream=True,
+        )
+
+        path: pathlib.Path = pathlib.Path(path)
+
+        ext = ".parquet"
+        if path.suffix != ext:
+            path = path.with_suffix(ext)
+
+        file = open(path, "wb")
+
+        chunk: bytes
+        for chunk in resp.iter_content(chunk_size=1024):
+            if chunk:
+                file.write(chunk)
+        file.close()
+
+    def query_as_json(
+        self, pipeline_name: str, query: str
+    ) -> Generator[dict, None, None]:
+        """
+        Executes an ad-hoc query on the specified pipeline and returns the result as a generator that yields
+        rows of the query as Python dictionaries.
+
+        :param pipeline_name: The name of the pipeline to query.
+        :param query: The SQL query to be executed.
+        :return: A generator that yields each row of the result as a Python dictionary, deserialized from JSON.
+        """
+        params = {
+            "pipeline_name": pipeline_name,
+            "sql": query,
+            "format": "json",
+        }
+
+        resp = self.http.get(
+            path=f"/pipelines/{pipeline_name}/query",
+            params=params,
+            stream=True,
+        )
+
+        for chunk in resp.iter_lines(chunk_size=50000000):
+            if chunk:
+                yield json.loads(chunk, parse_float=Decimal)

feldera/rest/pipeline.py

@@ -9,18 +9,22 @@ class Pipeline:
     """
 
     def __init__(
-            self,
-            name: str,
-            sql: str,
-            program_config: Mapping[str, Any],
-            runtime_config: Mapping[str, Any],
-            description: Optional[str] = None,
+        self,
+        name: str,
+        sql: str,
+        udf_rust: str,
+        udf_toml: str,
+        program_config: Mapping[str, Any],
+        runtime_config: Mapping[str, Any],
+        description: Optional[str] = None,
     ):
         """
         Initializes a new pipeline
 
         :param name: The name of the pipeline
         :param sql: The SQL code of the pipeline
+        :param udf_rust: Rust code for UDFs
+        :param udf_toml: Rust dependencies required by UDFs (in the TOML format)
         :param program_config: The program config of the pipeline
         :param runtime_config: The configuration of the pipeline
         :param description: Optional. The description of the pipeline
@@ -28,6 +32,8 @@ class Pipeline:
 
         self.name: str = name
         self.program_code: str = sql.strip()
+        self.udf_rust: str = udf_rust
+        self.udf_toml: str = udf_toml
         self.description: Optional[str] = description
         self.program_config: Mapping[str, Any] = program_config
         self.runtime_config: Mapping[str, Any] = runtime_config
@@ -44,13 +50,15 @@ class Pipeline:
         self.deployment_error: Optional[dict] = None
         self.deployment_location: Optional[str] = None
         self.program_binary_url: Optional[str] = None
-        self.program_info: Optional[dict] = None  # info about input & output connectors and the schema
+        self.program_info: Optional[dict] = (
+            None  # info about input & output connectors and the schema
+        )
         self.program_status: Optional[str] = None
         self.program_status_since: Optional[str] = None
 
     @classmethod
     def from_dict(cls, d: Mapping[str, Any]):
-        pipeline = cls("", "", {}, {})
+        pipeline = cls("", "", "", "", {}, {})
         pipeline.__dict__ = d
         pipeline.tables = []
         pipeline.views = []
@@ -58,11 +66,11 @@ class Pipeline:
         info = d.get("program_info")
 
         if info is not None:
-            for i in info['schema']['inputs']:
+            for i in info["schema"]["inputs"]:
                 tbl = SQLTable.from_dict(i)
                 pipeline.tables.append(tbl)
 
-            for output in info['schema']['outputs']:
+            for output in info["schema"]["outputs"]:
                 v = SQLView.from_dict(output)
                 pipeline.views.append(v)
 

feldera/rest/sql_table.py

@@ -3,7 +3,13 @@ class SQLTable:
     Represents a SQL table in Feldera
     """
 
-    def __init__(self, name: str, fields: list[dict], case_sensitive: bool = False, materialized: bool = False):
+    def __init__(
+        self,
+        name: str,
+        fields: list[dict],
+        case_sensitive: bool = False,
+        materialized: bool = False,
+    ):
         self.name = name
         self.case_sensitive = case_sensitive
         self.materialized = materialized
@@ -11,7 +17,7 @@ class SQLTable:
 
     @classmethod
     def from_dict(self, table_dict: dict):
-        tbl = SQLTable(name=table_dict['name'], fields=table_dict['fields'])
-        tbl.case_sensitive = table_dict['case_sensitive']
-        tbl.materialized = table_dict['materialized']
+        tbl = SQLTable(name=table_dict["name"], fields=table_dict["fields"])
+        tbl.case_sensitive = table_dict["case_sensitive"]
+        tbl.materialized = table_dict["materialized"]
         return tbl