feldera 0.134.0__tar.gz → 0.136.0__tar.gz

{feldera-0.134.0 → feldera-0.136.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.134.0
+Version: 0.136.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

{feldera-0.134.0 → feldera-0.136.0}/feldera/_callback_runner.py

@@ -39,14 +39,12 @@ class CallbackRunner(Thread):
         """
 
         pipeline = self.client.get_pipeline(self.pipeline_name)
-        schema = pipeline.program_info["schema"]
 
-        if schema:
-            schemas = [relation for relation in schema["inputs"] + schema["outputs"]]
-            for schema in schemas:
-                if schema["name"] == self.view_name:
-                    self.schema = schema
-                    break
+        schemas = pipeline.tables + pipeline.views
+        for schema in schemas:
+            if schema.name == self.view_name:
+                self.schema = schema
+                break
 
         if self.schema is None:
             raise ValueError(
@@ -66,7 +64,10 @@ class CallbackRunner(Thread):
             case _CallbackRunnerInstruction.PipelineStarted:
                 # listen to the pipeline
                 gen_obj = self.client.listen_to_pipeline(
-                    self.pipeline_name, self.view_name, format="json"
+                    self.pipeline_name,
+                    self.view_name,
+                    format="json",
+                    case_sensitive=self.schema.case_sensitive,
                 )
 
                 # if there is a queue set up, inform the main thread that the listener has been started, and it can
@@ -83,7 +84,7 @@ class CallbackRunner(Thread):
                     seq_no: Optional[int] = chunk.get("sequence_number")
                     if data is not None and seq_no is not None:
                         self.callback(
-                            dataframe_from_response([data], self.schema), seq_no
+                            dataframe_from_response([data], self.schema.fields), seq_no
                         )
 
                     if self.queue:

{feldera-0.134.0 → feldera-0.136.0}/feldera/_helpers.py

@@ -60,7 +60,7 @@ def ensure_dataframe_has_columns(df: pd.DataFrame):
         )
 
 
-def dataframe_from_response(buffer: list[list[dict]], schema: dict):
+def dataframe_from_response(buffer: list[list[dict]], fields: list[dict]):
     """
     Converts the response from Feldera to a pandas DataFrame.
     """
@@ -70,7 +70,7 @@ def dataframe_from_response(buffer: list[list[dict]], schema: dict):
     decimal_col = []
     uuid_col = []
 
-    for column in schema["fields"]:
+    for column in fields:
         column_name = column["name"]
         if not column["case_sensitive"]:
             column_name = column_name.lower()

{feldera-0.134.0 → feldera-0.136.0}/feldera/pipeline.py

@@ -685,6 +685,7 @@ pipeline '{self.name}' to make checkpoint '{seq}'"""
             checkpoint to complete syncing.
 
         :raises FelderaAPIError: If no checkpoints have been made.
+        :raises RuntimeError: If syncing the checkpoint fails.
         """
 
         uuid = self.client.sync_checkpoint(self.name)
@@ -702,6 +703,11 @@ pipeline '{self.name}' to make checkpoint '{seq}'"""
 pipeline '{self.name}' to sync checkpoint '{uuid}'"""
                 )
             status = self.sync_checkpoint_status(uuid)
+            if status == CheckpointStatus.Failure:
+                raise RuntimeError(
+                    f"failed to sync checkpoint '{uuid}': ", status.get_error()
+                )
+
             if status in [CheckpointStatus.InProgress, CheckpointStatus.Unknown]:
                 time.sleep(0.1)
                 continue
@@ -716,6 +722,9 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         If the checkpoint is currently being synchronized, returns
         `CheckpointStatus.Unknown`.
 
+        Failures are not raised as runtime errors and must be explicitly
+        checked.
+
         :param uuid: The checkpoint uuid.
         """
 
@@ -731,6 +740,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         if uuid == fail.get("uuid"):
             failure = CheckpointStatus.Failure
             failure.error = fail.get("error", "")
+            logging.error(f"failed to sync checkpoint '{uuid}': {failure.error}")
             return failure
 
         if (success is None) or UUID(uuid) > UUID(success):

{feldera-0.134.0 → feldera-0.136.0}/feldera/rest/_httprequests.py

@@ -26,7 +26,7 @@ class HttpRequests:
         self.headers = {"User-Agent": "feldera-python-sdk/v1"}
         self.requests_verify = config.requests_verify
 
-        if not self.requests_verify:
+        if isinstance(self.requests_verify, bool) and not self.requests_verify:
             urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
         if self.config.api_key:

{feldera-0.134.0 → feldera-0.136.0}/feldera/rest/config.py

@@ -1,4 +1,5 @@
 from typing import Optional
+import os
 
 
 class Config:
@@ -8,12 +9,12 @@ class Config:
 
     def __init__(
         self,
-        url: str,
+        url: Optional[str] = None,
         api_key: Optional[str] = None,
         version: Optional[str] = None,
         timeout: Optional[float] = None,
         connection_timeout: Optional[float] = None,
-        requests_verify: bool = True,
+        requests_verify: bool | str = True,
     ) -> None:
         """
         :param url: The url to the Feldera API (ex: https://try.feldera.com)
@@ -25,9 +26,22 @@ class Config:
             library. `True` by default.
         """
 
-        self.url: str = url
-        self.api_key: Optional[str] = api_key
+        BASE_URL = (
+            url
+            or os.environ.get("FELDERA_HOST")
+            or os.environ.get("FELDERA_BASE_URL")
+            or "http://localhost:8080"
+        )
+        self.url: str = BASE_URL
+        self.api_key: Optional[str] = os.environ.get("FELDERA_API_KEY", api_key)
         self.version: Optional[str] = version or "v0"
         self.timeout: Optional[float] = timeout
         self.connection_timeout: Optional[float] = connection_timeout
-        self.requests_verify: bool = requests_verify
+
+        FELDERA_TLS_INSECURE = True if os.environ.get("FELDERA_TLS_INSECURE") else False
+        FELDERA_HTTPS_TLS_CERT = os.environ.get("FELDERA_HTTPS_TLS_CERT")
+        requests_verify = not FELDERA_TLS_INSECURE
+        if requests_verify and FELDERA_HTTPS_TLS_CERT is not None:
+            requests_verify = FELDERA_HTTPS_TLS_CERT
+
+        self.requests_verify: bool | str = requests_verify

{feldera-0.134.0 → feldera-0.136.0}/feldera/rest/feldera_client.py

@@ -43,21 +43,24 @@ class FelderaClient:
 
     def __init__(
         self,
-        url: str,
+        url: Optional[str] = None,
         api_key: Optional[str] = None,
         timeout: Optional[float] = None,
         connection_timeout: Optional[float] = None,
-        requests_verify: bool = True,
+        requests_verify: bool | str = True,
     ) -> None:
         """
-        :param url: The url to Feldera API (ex: https://try.feldera.com)
+        :param url: The url to Feldera API (ex: https://try.feldera.com). If
+            not set, attempts to read from the environment variable
+            `FELDERA_HOST`. Default: `http://localhost:8080`
         :param api_key: The optional API key for Feldera
         :param timeout: (optional) The amount of time in seconds that the
             client will wait for a response before timing out.
-        :param connection_timeout: (optional) The amount of time in seconds that
-            the client will wait to establish connection before timing out.
+        :param connection_timeout: (optional) The amount of time in seconds
+            that the client will wait to establish connection before timing out
         :param requests_verify: The `verify` parameter passed to the requests
-            library. `True` by default.
+            library. `True` by default. To use a self signed certificate, set
+            it to the path to the certificate.
         """
 
         self.config = Config(
@@ -830,6 +833,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         backpressure: bool = True,
         array: bool = False,
         timeout: Optional[float] = None,
+        case_sensitive: bool = False,
     ):
         """
         Listen for updates to views for pipeline, yields the chunks of data
@@ -845,6 +849,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             "json" format, the default value is False
 
         :param timeout: The amount of time in seconds to listen to the stream for
+        :param case_sensitive: True if the table name is case sensitive or a reserved keyword, False by default
         """
 
         params = {
@@ -855,6 +860,8 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         if format == "json":
             params["array"] = _prepare_boolean_input(array)
 
+        table_name = f'"{table_name}"' if case_sensitive else table_name
+
         resp = self.http.post(
             path=f"/pipelines/{pipeline_name}/egress/{table_name}",
             params=params,
@@ -982,7 +989,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             stream=True,
         )
 
-        for chunk in resp.iter_lines(chunk_size=50000000):
+        for chunk in resp.iter_lines(chunk_size=1024):
             if chunk:
                 yield json.loads(chunk, parse_float=Decimal)
 

{feldera-0.134.0 → feldera-0.136.0}/feldera/testutils.py

@@ -23,6 +23,12 @@ KAFKA_SERVER = os.environ.get("FELDERA_KAFKA_SERVER", "localhost:19092")
 PIPELINE_TO_KAFKA_SERVER = os.environ.get(
     "FELDERA_PIPELINE_TO_KAFKA_SERVER", "redpanda:9092"
 )
+FELDERA_TLS_INSECURE = True if os.environ.get("FELDERA_TLS_INSECURE") else False
+FELDERA_HTTPS_TLS_CERT = os.environ.get("FELDERA_HTTPS_TLS_CERT")
+if not FELDERA_TLS_INSECURE and FELDERA_HTTPS_TLS_CERT is not None:
+    FELDERA_REQUESTS_VERIFY = FELDERA_HTTPS_TLS_CERT
+else:
+    FELDERA_REQUESTS_VERIFY = not FELDERA_TLS_INSECURE
 
 
 class _LazyClient:
@@ -36,7 +42,7 @@ class _LazyClient:
     def _ensure(self):
         if self._client is None:
             self._client = FelderaClient(
-                BASE_URL, api_key=API_KEY, connection_timeout=10, requests_verify=False
+                connection_timeout=10,
             )
         return self._client
 

{feldera-0.134.0 → feldera-0.136.0}/feldera.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.134.0
+Version: 0.136.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

{feldera-0.134.0 → feldera-0.136.0}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "feldera"
 readme = "README.md"
 description = "The feldera python client"
-version = "0.134.0"
+version = "0.136.0"
 license = { text = "MIT" }
 requires-python = ">=3.10"
 authors = [
@@ -45,3 +45,6 @@ dev-dependencies = [
     "sphinx==7.3.7",
     "simplejson==3.20.1"
 ]
+
+[tool.pytest.ini_options]
+pythonpath = "feldera"