feldera 0.111.0__tar.gz → 0.113.0__tar.gz

{feldera-0.111.0 → feldera-0.113.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.111.0
+Version: 0.113.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT
@@ -92,14 +92,14 @@ To run tests from a specific file:
 (cd python && python3 -m pytest ./tests/path-to-file.py)
 ```
 
-#### Running Aggregate Tests
+#### Running Tests
 
-The aggregate tests validate end-to-end correctness of SQL functionality.
-To run the aggregate tests use:
+The tests validate end-to-end correctness of SQL functionality.  To
+run the tests use:
 
 ```bash
 cd python
-PYTHONPATH=`pwd` python3 ./tests/aggregate_tests/main.py
+PYTHONPATH=`pwd` ./tests/run-all-tests.sh
 ```
 
 ### Reducing Compilation Cycles

{feldera-0.111.0 → feldera-0.113.0}/README.md

@@ -69,14 +69,14 @@ To run tests from a specific file:
 (cd python && python3 -m pytest ./tests/path-to-file.py)
 ```
 
-#### Running Aggregate Tests
+#### Running Tests
 
-The aggregate tests validate end-to-end correctness of SQL functionality.
-To run the aggregate tests use:
+The tests validate end-to-end correctness of SQL functionality.  To
+run the tests use:
 
 ```bash
 cd python
-PYTHONPATH=`pwd` python3 ./tests/aggregate_tests/main.py
+PYTHONPATH=`pwd` ./tests/run-all-tests.sh
 ```
 
 ### Reducing Compilation Cycles

{feldera-0.111.0 → feldera-0.113.0}/feldera/_callback_runner.py

@@ -75,7 +75,9 @@ class CallbackRunner(Thread):
                     # stop blocking the main thread on `join` for the previous message
                     self.queue.task_done()
 
-                for chunk in gen_obj:
+                iterator = gen_obj()
+
+                for chunk in iterator:
                     chunk: dict = chunk
                     data: Optional[list[dict]] = chunk.get("json_data")
                     seq_no: Optional[int] = chunk.get("sequence_number")

{feldera-0.111.0 → feldera-0.113.0}/feldera/pipeline.py

@@ -484,7 +484,9 @@ metrics"""
             for view_name, queue in self.views_tx.pop().items():
                 # block until the callback runner has been stopped
                 queue.join()
+        import time
 
+        time.sleep(3)
         self.client.stop_pipeline(self.name, force=force, timeout_s=timeout_s)
 
     def resume(self, timeout_s: Optional[float] = None):
@@ -838,14 +840,14 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
 
     def set_runtime_config(self, runtime_config: RuntimeConfig):
         """Updates the runtime config of the pipeline.  The pipeline
-        must be stopped and, in addition, changing some pipeline
-        configuration requires storage to be cleared.
+        must be stopped.  Changing some pipeline configuration, such
+        as the number of workers, requires storage to be cleared.
 
-        For example, to set 'min_batch_size_records' on a pipeline:
+        For example, to set 'min_batch_size_records' on a pipeline::
 
-        runtime_config = pipeline.runtime_config()
-        runtime_config.min_batch_size_records = 500
-        pipeline.set_runtime_config(runtime_config)
+            runtime_config = pipeline.runtime_config()
+            runtime_config.min_batch_size_records = 500
+            pipeline.set_runtime_config(runtime_config)
 
         """
 

{feldera-0.111.0 → feldera-0.113.0}/feldera/rest/_httprequests.py

@@ -12,6 +12,7 @@ import json
 import requests
 from requests.packages import urllib3
 from typing import Callable, Optional, Any, Union, Mapping, Sequence, List
+import time
 
 
 def json_serialize(body: Any) -> str:
@@ -42,6 +43,7 @@ class HttpRequests:
         params: Optional[Mapping[str, Any]] = None,
         stream: bool = False,
         serialize: bool = True,
+        max_retries: int = 3,
     ) -> Any:
         """
         :param http_method: The HTTP method to use. Takes the equivalent `requests.*` module. (Example: `requests.get`)
@@ -68,39 +70,54 @@ class HttpRequests:
                 str(params),
             )
 
-            if http_method.__name__ == "get":
-                request = http_method(
-                    request_path,
-                    timeout=timeout,
-                    headers=headers,
-                    params=params,
-                    stream=stream,
-                    verify=self.requests_verify,
-                )
-            elif isinstance(body, bytes):
-                request = http_method(
-                    request_path,
-                    timeout=timeout,
-                    headers=headers,
-                    data=body,
-                    params=params,
-                    stream=stream,
-                    verify=self.requests_verify,
-                )
-            else:
-                request = http_method(
-                    request_path,
-                    timeout=timeout,
-                    headers=headers,
-                    data=json_serialize(body) if serialize else body,
-                    params=params,
-                    stream=stream,
-                    verify=self.requests_verify,
-                )
-
-            resp = self.__validate(request, stream=stream)
-            logging.debug("got response: %s", str(resp))
-            return resp
+            for attempt in range(max_retries):
+                if http_method.__name__ == "get":
+                    request = http_method(
+                        request_path,
+                        timeout=timeout,
+                        headers=headers,
+                        params=params,
+                        stream=stream,
+                        verify=self.requests_verify,
+                    )
+                elif isinstance(body, bytes):
+                    request = http_method(
+                        request_path,
+                        timeout=timeout,
+                        headers=headers,
+                        data=body,
+                        params=params,
+                        stream=stream,
+                        verify=self.requests_verify,
+                    )
+                else:
+                    request = http_method(
+                        request_path,
+                        timeout=timeout,
+                        headers=headers,
+                        data=json_serialize(body) if serialize else body,
+                        params=params,
+                        stream=stream,
+                        verify=self.requests_verify,
+                    )
+
+                try:
+                    resp = self.__validate(request, stream=stream)
+                    logging.debug("got response: %s", str(resp))
+                    return resp
+                except FelderaAPIError as err:
+                    # Only retry on 503
+                    if err.status_code == 503:
+                        if attempt < max_retries:
+                            logging.warning(
+                                "HTTP 503 received for %s, retrying (%d/%d)...",
+                                path,
+                                attempt + 1,
+                                max_retries,
+                            )
+                            time.sleep(2)  # backoff, adjust as needed
+                            continue
+                    raise  # re-raise for all other errors or if out of retries
 
         except requests.exceptions.Timeout as err:
             raise FelderaTimeoutError(str(err)) from err

{feldera-0.111.0 → feldera-0.113.0}/feldera/rest/feldera_client.py

@@ -272,7 +272,11 @@ class FelderaClient:
 
             if status == "Running":
                 break
-            elif status == "Failed":
+            elif (
+                status == "Stopped"
+                and len(resp.deployment_error or {}) > 0
+                and resp.deployment_desired_status == "Stopped"
+            ):
                 raise RuntimeError(
                     f"""Unable to START the pipeline.
 Reason: The pipeline is in a STOPPED state due to the following error:
@@ -601,13 +605,16 @@ Reason: The pipeline is in a STOPPED state due to the following error:
 
         end = time.monotonic() + timeout if timeout else None
 
-        # Using the default chunk size below makes `iter_lines` extremely
-        # inefficient when dealing with long lines.
-        for chunk in resp.iter_lines(chunk_size=50000000):
-            if end and time.monotonic() > end:
-                break
-            if chunk:
-                yield json.loads(chunk, parse_float=Decimal)
+        def generator():
+            # Using the default chunk size below makes `iter_lines` extremely
+            # inefficient when dealing with long lines.
+            for chunk in resp.iter_lines(chunk_size=50000000):
+                if end and time.monotonic() > end:
+                    break
+                if chunk:
+                    yield json.loads(chunk, parse_float=Decimal)
+
+        return generator
 
     def query_as_text(
         self, pipeline_name: str, query: str

{feldera-0.111.0 → feldera-0.113.0}/feldera.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.111.0
+Version: 0.113.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT
@@ -92,14 +92,14 @@ To run tests from a specific file:
 (cd python && python3 -m pytest ./tests/path-to-file.py)
 ```
 
-#### Running Aggregate Tests
+#### Running Tests
 
-The aggregate tests validate end-to-end correctness of SQL functionality.
-To run the aggregate tests use:
+The tests validate end-to-end correctness of SQL functionality.  To
+run the tests use:
 
 ```bash
 cd python
-PYTHONPATH=`pwd` python3 ./tests/aggregate_tests/main.py
+PYTHONPATH=`pwd` ./tests/run-all-tests.sh
 ```
 
 ### Reducing Compilation Cycles

{feldera-0.111.0 → feldera-0.113.0}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "feldera"
 readme = "README.md"
 description = "The feldera python client"
-version = "0.111.0"
+version = "0.113.0"
 license = { text = "MIT" }
 requires-python = ">=3.10"
 authors = [

{feldera-0.111.0 → feldera-0.113.0}/tests/test_shared_pipeline0.py

@@ -1,6 +1,5 @@
 import os
 import pathlib
-import threading
 import pandas as pd
 import time
 import unittest
@@ -68,35 +67,6 @@ class TestPipeline(SharedTestPipeline):
         TEST_CLIENT.pause_pipeline(self.pipeline.name)
         TEST_CLIENT.stop_pipeline(self.pipeline.name, force=True)
 
-    def __listener(self):
-        gen_obj = TEST_CLIENT.listen_to_pipeline(
-            pipeline_name=self.pipeline.name,
-            table_name="v0",
-            format="csv",
-        )
-        counter = 0
-        for chunk in gen_obj:
-            counter += 1
-            text_data = chunk.get("text_data")
-            if text_data:
-                assert text_data == "1,1\n2,1\n"
-                self.result = True
-                break
-            if counter > 10:
-                self.result = False
-                break
-
-    def test_listen_to_pipeline(self):
-        data = "1\n2\n"
-        TEST_CLIENT.pause_pipeline(self.pipeline.name)
-        t1 = threading.Thread(target=self.__listener)
-        t1.start()
-        self.pipeline.resume()
-        TEST_CLIENT.push_to_pipeline(self.pipeline.name, "tbl", "csv", data)
-        t1.join()
-        assert self.result
-        TEST_CLIENT.stop_pipeline(self.pipeline.name, force=True)
-
     def test_adhoc_query_text(self):
         data = "1\n2\n"
         self.pipeline.start()
@@ -567,6 +537,7 @@ class TestPipeline(SharedTestPipeline):
         with self.assertRaises(ValueError):
             data = {"m_var": {None: 1}}
             self.pipeline.input_json("tbl_map_issue3754", [data])
+        self.pipeline.stop(force=True)
 
     def test_pipeline_resource_config(self):
         from feldera.runtime_config import Resources, RuntimeConfig

{feldera-0.111.0 → feldera-0.113.0}/tests/test_shared_pipeline1.py

@@ -1,12 +1,16 @@
 import random
+from uuid import uuid4
 import time
+import os
 from typing import Optional
 from feldera.runtime_config import RuntimeConfig, Storage
 from tests import enterprise_only
 from tests.shared_test_pipeline import SharedTestPipeline
 
 
-DEFAULT_ENDPOINT = "http://minio.extra.svc.cluster.local:9000"
+DEFAULT_ENDPOINT = os.environ.get(
+    "DEFAULT_MINIO_ENDPOINT", "http://minio.extra.svc.cluster.local:9000"
+)
 DEFAULT_BUCKET = "default"
 ACCESS_KEY = "minio"
 SECRET_KEY = "miniopasswd"
@@ -36,7 +40,13 @@ def storage_cfg(
 
 class TestCheckpointSync(SharedTestPipeline):
     @enterprise_only
-    def test_checkpoint_sync(self, from_uuid: bool = False, auth_err: bool = False):
+    def test_checkpoint_sync(
+        self,
+        from_uuid: bool = False,
+        random_uuid: bool = False,
+        clear_storage: bool = True,
+        auth_err: bool = False,
+    ):
         """
         CREATE TABLE t0 (c0 INT, c1 VARCHAR);
         CREATE MATERIALIZED VIEW v0 AS SELECT c0 FROM t0;
@@ -56,7 +66,12 @@ class TestCheckpointSync(SharedTestPipeline):
         uuid = self.pipeline.sync_checkpoint(wait=True)
 
         self.pipeline.stop(force=True)
-        self.pipeline.clear_storage()
+
+        if clear_storage:
+            self.pipeline.clear_storage()
+
+        if random_uuid:
+            uuid = uuid4()
 
         # Restart pipeline from checkpoint
         storage_config = storage_cfg(
@@ -69,13 +84,24 @@ class TestCheckpointSync(SharedTestPipeline):
         self.assertCountEqual(got_before, got_after)
 
         self.pipeline.stop(force=True)
-        self.pipeline.clear_storage()
+
+        if clear_storage:
+            self.pipeline.clear_storage()
 
     @enterprise_only
     def test_checkpoint_sync_from_uuid(self):
         self.test_checkpoint_sync(from_uuid=True)
 
+    @enterprise_only
+    def test_checkpoint_sync_without_clearing_storage(self):
+        self.test_checkpoint_sync(clear_storage=False)
+
     @enterprise_only
     def test_checkpoint_sync_err(self):
         with self.assertRaisesRegex(RuntimeError, "SignatureDoesNotMatch"):
             self.test_checkpoint_sync(auth_err=True)
+
+    @enterprise_only
+    def test_checkpoint_sync_err_nonexistent_checkpoint(self):
+        with self.assertRaisesRegex(RuntimeError, "were not found in source"):
+            self.test_checkpoint_sync(random_uuid=True, from_uuid=True)