feldera 0.100.0__py3-none-any.whl → 0.101.0__py3-none-any.whl

feldera/_callback_runner.py

@@ -54,12 +54,12 @@ class CallbackRunner(Thread):
             )
 
         # by default, we assume that the pipeline has been started
-        ack: _CallbackRunnerInstruction = _CallbackRunnerInstruction.PipelineStarted
+        ack = _CallbackRunnerInstruction.PipelineStarted
 
         # if there is Queue, we wait for the instruction to start the pipeline
         # this means that we are listening to the pipeline before running it, therefore, all data should be received
         if self.queue:
-            ack: _CallbackRunnerInstruction = self.queue.get()
+            ack = self.queue.get()
 
         match ack:
             # if the pipeline has actually been started, we start a listener
@@ -77,11 +77,12 @@ class CallbackRunner(Thread):
 
                 for chunk in gen_obj:
                     chunk: dict = chunk
-                    data: list[dict] = chunk.get("json_data")
-                    seq_no: int = chunk.get("sequence_number")
-
-                    if data is not None:
-                        self.callback(dataframe_from_response([data], schema), seq_no)
+                    data: Optional[list[dict]] = chunk.get("json_data")
+                    seq_no: Optional[int] = chunk.get("sequence_number")
+                    if data is not None and seq_no is not None:
+                        self.callback(
+                            dataframe_from_response([data], self.schema), seq_no
+                        )
 
                     if self.queue:
                         try:

feldera/pipeline.py

@@ -473,10 +473,13 @@ metrics"""
             pipeline to stop.
         """
 
-        if len(self.views_tx) > 0:
-            for _, queue in self.views_tx.pop().items():
+        for view_queue in self.views_tx:
+            for _, queue in view_queue.items():
                 # sends a message to the callback runner to stop listening
                 queue.put(_CallbackRunnerInstruction.RanToCompletion)
+
+        if len(self.views_tx) > 0:
+            for view_name, queue in self.views_tx.pop().items():
                 # block until the callback runner has been stopped
                 queue.join()
 

feldera/pipeline_builder.py

@@ -2,7 +2,7 @@ from feldera.rest.feldera_client import FelderaClient
 from feldera.rest.pipeline import Pipeline as InnerPipeline
 from feldera.pipeline import Pipeline
 from feldera.enums import CompilationProfile
-from feldera.runtime_config import RuntimeConfig, Resources
+from feldera.runtime_config import RuntimeConfig
 from feldera.rest.errors import FelderaAPIError
 
 
@@ -29,7 +29,7 @@ class PipelineBuilder:
         udf_toml: str = "",
         description: str = "",
         compilation_profile: CompilationProfile = CompilationProfile.OPTIMIZED,
-        runtime_config: RuntimeConfig = RuntimeConfig(resources=Resources()),
+        runtime_config: RuntimeConfig = RuntimeConfig.default(),
     ):
         self.client: FelderaClient = client
         self.name: str | None = name

feldera/rest/feldera_client.py

@@ -454,11 +454,11 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         pipeline_name: str,
         table_name: str,
         format: str,
-        data: list[list | str | dict] | dict,
+        data: list[list | str | dict] | dict | str,
         array: bool = False,
         force: bool = False,
         update_format: str = "raw",
-        json_flavor: str = None,
+        json_flavor: Optional[str] = None,
         serialize: bool = True,
     ):
         """

feldera/runtime_config.py

@@ -1,3 +1,4 @@
+import os
 from typing import Optional, Any, Mapping
 
 
@@ -72,6 +73,7 @@ class RuntimeConfig:
         clock_resolution_usecs: Optional[int] = None,
         provisioning_timeout_secs: Optional[int] = None,
         resources: Optional[Resources] = None,
+        runtime_version: Optional[str] = None,
     ):
         self.workers = workers
         self.tracing = tracing
@@ -81,6 +83,9 @@ class RuntimeConfig:
         self.min_batch_size_records = min_batch_size_records
         self.clock_resolution_usecs = clock_resolution_usecs
         self.provisioning_timeout_secs = provisioning_timeout_secs
+        self.runtime_version = runtime_version or os.environ.get(
+            "FELDERA_RUNTIME_VERSION"
+        )
         if resources is not None:
             self.resources = resources.__dict__
         if isinstance(storage, bool):
@@ -88,6 +93,10 @@ class RuntimeConfig:
         if isinstance(storage, Storage):
             self.storage = storage.__dict__
 
+    @staticmethod
+    def default() -> "RuntimeConfig":
+        return RuntimeConfig(resources=Resources())
+
     @classmethod
     def from_dict(cls, d: Mapping[str, Any]):
         """

{feldera-0.100.0.dist-info → feldera-0.101.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.100.0
+Version: 0.101.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT
@@ -54,10 +54,11 @@ If you have cloned the Feldera repo, you can install the python SDK as follows:
 pip install python/
 ```
 
-Checkout the docs [here](./feldera/__init__.py) for an example on how to use the SDK.
-
 ## Documentation
 
+The Python SDK documentation is available at
+[Feldera Python SDK Docs](https://docs.feldera.com/python).
+
 To build the html documentation run:
 
 Ensure that you have sphinx installed. If not, install it using `pip install sphinx`.
@@ -77,27 +78,23 @@ To clean the build, run `make clean`.
 To run unit tests:
 
 ```bash
-(cd python && python3 -m unittest)
+cd python && python3 -m pytest tests/
 ```
 
-> ⚠️ Running the unit tests will **delete all existing pipelines**.
-
-The following command runs end-to-end tests.  You'll need a pipeline
-manager running at `http://localhost:8080`.  For the pipeline builder
-tests, you'll also need a broker available at `localhost:9092` and
-(from the pipelines) `redpanda:19092`.  (To change those locations,
-set the environment variables listed in `python/tests/__init__.py`.)
-
-```bash
-(cd python && python3 -m pytest tests)
-```
+- This will detect and run all test files that match the pattern `test_*.py` or
+  `*_test.py`.
+- By default, the tests expect a running Feldera instance at `http://localhost:8080`.
+  To override the default endpoint, set the `FELDERA_BASE_URL` environment variable.
 
 To run tests from a specific file:
 
 ```bash
-(cd python && python3 -m unittest ./tests/path-to-file.py)
+(cd python && python3 -m pytest ./tests/path-to-file.py)
 ```
 
+#### Running Aggregate Tests
+
+The aggregate tests validate end-to-end correctness of SQL functionality.
 To run the aggregate tests use:
 
 ```bash
@@ -105,6 +102,38 @@ cd python
 PYTHONPATH=`pwd` python3 ./tests/aggregate_tests/main.py
 ```
 
+### Reducing Compilation Cycles
+
+To reduce redundant compilation cycles during testing:
+
+* **Inherit from `SharedTestPipeline`** instead of `unittest.TestCase`.
+* **Define DDLs** (e.g., `CREATE TABLE`, `CREATE VIEW`) in the **docstring** of each test method.
+  * All DDLs from all test functions in the class are combined and compiled into a single pipeline.
+  * If a table or view is already defined in one test, it can be used directly in others without redefinition.
+  * Ensure that all table and view names are unique within the class.
+* Use `@enterprise_only` on tests that require Enterprise features. Their DDLs will be skipped on OSS builds.
+* Use `self.set_runtime_config(...)` to override the default pipeline config.
+  * Reset it at the end using `self.reset_runtime_config()`.
+* Access the shared pipeline via `self.pipeline`.
+
+#### Example
+
+```python
+from tests.shared_test_pipeline import SharedTestPipeline
+
+class TestAverage(SharedTestPipeline):
+    def test_average(self):
+        """
+        CREATE TABLE students(id INT, name STRING);
+        CREATE MATERIALIZED VIEW v AS SELECT * FROM students;
+        """
+        ...
+        self.pipeline.start()
+        self.pipeline.input_pandas("students", df)
+        self.pipeline.wait_for_completion(True)
+        ...
+```
+
 ## Linting and formatting
 
 Use [Ruff] to run the lint checks that will be executed by the

{feldera-0.100.0.dist-info → feldera-0.101.0.dist-info}/RECORD

@@ -1,23 +1,23 @@
 feldera/__init__.py,sha256=EiY3bTj_mnfNhCGrZo6J__brfovIJ-YYAdy77PyaEoo,378
-feldera/_callback_runner.py,sha256=Tdf6BXN4zppyoy8t_y-Ooa3B0wEfvyezMHU9jxY2ZhA,4713
+feldera/_callback_runner.py,sha256=GNOg3TrKJg9zJU0HvpWxCHqzjMUX8ORiHhtiEEdVQzE,4758
 feldera/_helpers.py,sha256=rN0WuGSCCQlXWFMimZUQrgs-LJAfUo074d79sLElncQ,3023
 feldera/enums.py,sha256=AWx7wldY2vBbrJMAwFabGQ8C_ZTKZreiqRvuGlrC9Ks,8702
 feldera/output_handler.py,sha256=64J3ljhOaKIhxdjOKYi-BUz_HnMwROfmN8eE-btYygU,1930
-feldera/pipeline.py,sha256=5-OP_ugda-_BKhB3232JEHrzkO-FFuZtjVjLJ3yJ2Bg,34309
-feldera/pipeline_builder.py,sha256=xOIQhkTJ_E9gMCLI-4_xcGwbUdoq-aT9UK_khj0eiog,3948
-feldera/runtime_config.py,sha256=EDvnakyaRTHUS8GLd6TqLZ_jsGp7_4fbcPnSfQCw1k0,3501
+feldera/pipeline.py,sha256=Wc9yfnEXkVQ_WQSYuENisN6VSzANVJbsSuQGt1K0E_k,34407
+feldera/pipeline_builder.py,sha256=rImmAkYgpCGxZyH317mUMvChLuGF0rH8gvEOVcTYN4M,3924
+feldera/runtime_config.py,sha256=jeVuv2B6H0IDSmkyQgsuSOv1B656jvJIL9re4Sv4Rjw,3781
 feldera/stats.py,sha256=XBhkRsV7FXErwWuPP0i3q9W77mzkMo-oThPVEZy5y3U,5028
 feldera/rest/__init__.py,sha256=Eg-EKUU3RSTDcdxTR_7wNDnCly8VpXEzsZCQUmf-y2M,308
 feldera/rest/_helpers.py,sha256=q7jWInKp9IiIli8N5o31lDG3hNUbcsJqufZXYHG04ps,222
 feldera/rest/_httprequests.py,sha256=e22YbpzOzy7MGo7hk9MOU7ZRTj3F314grY0Ygr-_goI,6636
 feldera/rest/config.py,sha256=DYzZKngDEhouTEwqVFd-rDrBN9tWqsU07Jl_BTT4mXs,1008
 feldera/rest/errors.py,sha256=b4i2JjrbSmej7jdko_FL8UeXklLKenSipwMT80jowaM,1720
-feldera/rest/feldera_client.py,sha256=xWmoHUm-epaDZP64Wgc4PXCWSPwfSXoZrIF_93WIYok,25273
+feldera/rest/feldera_client.py,sha256=E0_cqpIaavVeapBLc5egZIl5KVsKWliDzWqXk6_r0nY,25289
 feldera/rest/feldera_config.py,sha256=1pnGbLFMSLvp7Qh_OlPLALSKCSHIktNWKvx6gYU00U4,1374
 feldera/rest/pipeline.py,sha256=-dGGUdtHMABKrQEclaeuwGI_FOCrQOk6p2aCFV0FdU8,2890
 feldera/rest/sql_table.py,sha256=qrw-YwMzx5T81zDefNO1KOx7EyypFz1vPwGBzSUB7kc,652
 feldera/rest/sql_view.py,sha256=hN12mPM0mvwLCIPYywpb12s9Hd2Ws31IlTMXPriMisw,644
-feldera-0.100.0.dist-info/METADATA,sha256=FXjoSJakdfYI6Um1ZdZJnM8mfzjXiee4nYn5Gd8Rzjc,2893
-feldera-0.100.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-feldera-0.100.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
-feldera-0.100.0.dist-info/RECORD,,
+feldera-0.101.0.dist-info/METADATA,sha256=6H0v0BPG56ovszTNICNFycsdWM26sDt3Mdsyp4dQ3_4,4101
+feldera-0.101.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+feldera-0.101.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
+feldera-0.101.0.dist-info/RECORD,,