feldera 0.99.0__py3-none-any.whl → 0.101.0__py3-none-any.whl

feldera/__init__.py

@@ -1,6 +1,9 @@
 from feldera.rest.feldera_client import FelderaClient as FelderaClient
 from feldera.pipeline import Pipeline as Pipeline
 from feldera.pipeline_builder import PipelineBuilder as PipelineBuilder
+from feldera.rest._helpers import client_version
+
+__version__ = client_version()
 
 import pretty_errors
 

feldera/_callback_runner.py

@@ -54,12 +54,12 @@ class CallbackRunner(Thread):
             )
 
         # by default, we assume that the pipeline has been started
-        ack: _CallbackRunnerInstruction = _CallbackRunnerInstruction.PipelineStarted
+        ack = _CallbackRunnerInstruction.PipelineStarted
 
         # if there is Queue, we wait for the instruction to start the pipeline
         # this means that we are listening to the pipeline before running it, therefore, all data should be received
         if self.queue:
-            ack: _CallbackRunnerInstruction = self.queue.get()
+            ack = self.queue.get()
 
         match ack:
             # if the pipeline has actually been started, we start a listener
@@ -77,11 +77,12 @@ class CallbackRunner(Thread):
 
                 for chunk in gen_obj:
                     chunk: dict = chunk
-                    data: list[dict] = chunk.get("json_data")
-                    seq_no: int = chunk.get("sequence_number")
-
-                    if data is not None:
-                        self.callback(dataframe_from_response([data], schema), seq_no)
+                    data: Optional[list[dict]] = chunk.get("json_data")
+                    seq_no: Optional[int] = chunk.get("sequence_number")
+                    if data is not None and seq_no is not None:
+                        self.callback(
+                            dataframe_from_response([data], self.schema), seq_no
+                        )
 
                     if self.queue:
                         try:

feldera/enums.py

@@ -36,27 +36,28 @@ class BuildMode(Enum):
 
 class PipelineStatus(Enum):
     """
-      Represents the state that this pipeline is currently in.
-
-      .. code-block:: text
-
-                  Stopped ◄─────────── Stopping ◄───── All states can transition
-                     │                    ▲            to Stopping by either:
-    /start or /pause │                    │            (1) user calling /stop?force=true, or;
-                     ▼                    │            (2) pipeline encountering a fatal
-              ⌛Provisioning          Suspending            resource or runtime error,
-                     │                    ▲                having the system call /stop?force=true
-                     ▼                    │ /stop          effectively
-              ⌛Initializing ─────────────┤  ?force=false
-                     │                    │
-           ┌─────────┼────────────────────┴─────┐
-           │         ▼                          │
-           │       Paused  ◄──────► Unavailable │
-           │        │   ▲                ▲      │
-           │ /start │   │  /pause        │      │
-           │        ▼   │                │      │
-           │       Running ◄─────────────┘      │
-           └────────────────────────────────────┘
+    Represents the state that this pipeline is currently in.
+
+    .. code-block:: text
+
+                    Stopped ◄─────────── Stopping ◄───── All states can transition
+                       │                    ▲            to Stopping by either:
+      /start or /pause │                    │            (1) user calling /stop?force=true, or;
+                       ▼                    │            (2) pipeline encountering a fatal
+                ⌛Provisioning          Suspending            resource or runtime error,
+                       │                    ▲                having the system call /stop?force=true
+                       ▼                    │ /stop          effectively
+                ⌛Initializing ─────────────┤  ?force=false
+                       │                    │
+             ┌─────────┼────────────────────┴─────┐
+             │         ▼                          │
+             │       Paused  ◄──────► Unavailable │
+             │        │   ▲                ▲      │
+             │ /start │   │  /pause        │      │
+             │        ▼   │                │      │
+             │       Running ◄─────────────┘      │
+             └────────────────────────────────────┘
+
     """
 
     NOT_FOUND = 0

feldera/pipeline.py

@@ -111,9 +111,7 @@ class Pipeline:
             tbl.name.lower() for tbl in pipeline.tables
         ]:
             raise ValueError(
-                f"Cannot push to table '{
-                    table_name
-                }': table with this name does not exist in the '{self.name}' pipeline"
+                f"Cannot push to table '{table_name}': table with this name does not exist in the '{self.name}' pipeline"
             )
         else:
             # consider validating the schema here
@@ -308,14 +306,10 @@ class Pipeline:
                 elapsed = time.monotonic() - start_time
                 if elapsed > timeout_s:
                     raise TimeoutError(
-                        f"timeout ({timeout_s}s) reached while waiting for pipeline '{
-                            self.name
-                        }' to complete"
+                        f"timeout ({timeout_s}s) reached while waiting for pipeline '{self.name}' to complete"
                     )
                 logging.debug(
-                    f"waiting for pipeline {self.name} to complete: elapsed time {
-                        elapsed
-                    }s, timeout: {timeout_s}s"
+                    f"waiting for pipeline {self.name} to complete: elapsed time {elapsed}s, timeout: {timeout_s}s"
                 )
 
             pipeline_complete: bool = self.stats().global_metrics.pipeline_complete
@@ -402,15 +396,11 @@ method or use `Pipeline.resume()` to resume a paused pipeline."""
         """
         if idle_interval_s > timeout_s:
             raise ValueError(
-                f"idle interval ({idle_interval_s}s) cannot be larger than timeout ({
-                    timeout_s
-                }s)"
+                f"idle interval ({idle_interval_s}s) cannot be larger than timeout ({timeout_s}s)"
             )
         if poll_interval_s > timeout_s:
             raise ValueError(
-                f"poll interval ({poll_interval_s}s) cannot be larger than timeout ({
-                    timeout_s
-                }s)"
+                f"poll interval ({poll_interval_s}s) cannot be larger than timeout ({timeout_s}s)"
             )
         if poll_interval_s > idle_interval_s:
             raise ValueError(
@@ -483,10 +473,13 @@ metrics"""
             pipeline to stop.
         """
 
-        if len(self.views_tx) > 0:
-            for _, queue in self.views_tx.pop().items():
+        for view_queue in self.views_tx:
+            for _, queue in view_queue.items():
                 # sends a message to the callback runner to stop listening
                 queue.put(_CallbackRunnerInstruction.RanToCompletion)
+
+        if len(self.views_tx) > 0:
+            for view_name, queue in self.views_tx.pop().items():
                 # block until the callback runner has been stopped
                 queue.join()
 

feldera/pipeline_builder.py

@@ -2,7 +2,7 @@ from feldera.rest.feldera_client import FelderaClient
 from feldera.rest.pipeline import Pipeline as InnerPipeline
 from feldera.pipeline import Pipeline
 from feldera.enums import CompilationProfile
-from feldera.runtime_config import RuntimeConfig, Resources
+from feldera.runtime_config import RuntimeConfig
 from feldera.rest.errors import FelderaAPIError
 
 
@@ -29,7 +29,7 @@ class PipelineBuilder:
         udf_toml: str = "",
         description: str = "",
         compilation_profile: CompilationProfile = CompilationProfile.OPTIMIZED,
-        runtime_config: RuntimeConfig = RuntimeConfig(resources=Resources()),
+        runtime_config: RuntimeConfig = RuntimeConfig.default(),
     ):
         self.client: FelderaClient = client
         self.name: str | None = name
@@ -50,8 +50,12 @@ class PipelineBuilder:
         if self.name is None or self.sql is None:
             raise ValueError("Name and SQL are required to create a pipeline")
 
-        if self.client.get_pipeline(self.name) is not None:
-            raise RuntimeError(f"Pipeline with name {self.name} already exists")
+        try:
+            if self.client.get_pipeline(self.name) is not None:
+                raise RuntimeError(f"Pipeline with name {self.name} already exists")
+        except FelderaAPIError as err:
+            if err.error_code != "UnknownPipelineName":
+                raise err
 
         inner = InnerPipeline(
             self.name,
@@ -62,7 +66,9 @@ class PipelineBuilder:
             program_config={
                 "profile": self.compilation_profile.value,
             },
-            runtime_config=self.runtime_config.__dict__,
+            runtime_config=dict(
+                (k, v) for k, v in self.runtime_config.__dict__.items() if v is not None
+            ),
         )
 
         inner = self.client.create_pipeline(inner)

feldera/rest/_helpers.py

@@ -0,0 +1,9 @@
+def client_version() -> str:
+    from importlib.metadata import version, PackageNotFoundError
+
+    try:
+        version = version("feldera")
+    except PackageNotFoundError:
+        version = "unknown"
+
+    return version

feldera/rest/feldera_client.py

@@ -11,6 +11,7 @@ from feldera.rest.feldera_config import FelderaConfig
 from feldera.rest.errors import FelderaTimeoutError
 from feldera.rest.pipeline import Pipeline
 from feldera.rest._httprequests import HttpRequests
+from feldera.rest._helpers import client_version
 
 
 def _validate_no_none_keys_in_map(data):
@@ -63,7 +64,13 @@ class FelderaClient:
         self.http = HttpRequests(self.config)
 
         try:
-            self.pipelines()
+            config = self.get_config()
+            version = client_version()
+            if config.version != version:
+                logging.warn(
+                    f"Client is on version {version} while server is at "
+                    f"{config.version}. There could be incompatibilities."
+                )
         except Exception as e:
             logging.error(f"Failed to connect to Feldera API: {e}")
             raise e
@@ -358,9 +365,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             time.sleep(0.1)
 
         raise FelderaTimeoutError(
-            f"timeout error: pipeline '{pipeline_name}' did not stop in {
-                timeout_s
-            } seconds"
+            f"timeout error: pipeline '{pipeline_name}' did not stop in {timeout_s} seconds"
         )
 
     def clear_storage(self, pipeline_name: str, timeout_s: Optional[float] = 300):
@@ -394,9 +399,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
             time.sleep(0.1)
 
         raise FelderaTimeoutError(
-            f"timeout error: pipeline '{pipeline_name}' did not clear storage in {
-                timeout_s
-            } seconds"
+            f"timeout error: pipeline '{pipeline_name}' did not clear storage in {timeout_s} seconds"
         )
 
     def checkpoint_pipeline(self, pipeline_name: str) -> int:
@@ -451,11 +454,11 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         pipeline_name: str,
         table_name: str,
         format: str,
-        data: list[list | str | dict] | dict,
+        data: list[list | str | dict] | dict | str,
         array: bool = False,
         force: bool = False,
         update_format: str = "raw",
-        json_flavor: str = None,
+        json_flavor: Optional[str] = None,
         serialize: bool = True,
     ):
         """
@@ -702,9 +705,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         """
 
         self.http.post(
-            path=f"/pipelines/{pipeline_name}/tables/{table_name}/connectors/{
-                connector_name
-            }/pause",
+            path=f"/pipelines/{pipeline_name}/tables/{table_name}/connectors/{connector_name}/pause",
         )
 
     def resume_connector(
@@ -728,9 +729,7 @@ Reason: The pipeline is in a STOPPED state due to the following error:
         """
 
         self.http.post(
-            path=f"/pipelines/{pipeline_name}/tables/{table_name}/connectors/{
-                connector_name
-            }/start",
+            path=f"/pipelines/{pipeline_name}/tables/{table_name}/connectors/{connector_name}/start",
         )
 
     def get_config(self) -> FelderaConfig:

feldera/runtime_config.py

@@ -1,3 +1,4 @@
+import os
 from typing import Optional, Any, Mapping
 
 
@@ -72,6 +73,7 @@ class RuntimeConfig:
         clock_resolution_usecs: Optional[int] = None,
         provisioning_timeout_secs: Optional[int] = None,
         resources: Optional[Resources] = None,
+        runtime_version: Optional[str] = None,
     ):
         self.workers = workers
         self.tracing = tracing
@@ -81,6 +83,9 @@ class RuntimeConfig:
         self.min_batch_size_records = min_batch_size_records
         self.clock_resolution_usecs = clock_resolution_usecs
         self.provisioning_timeout_secs = provisioning_timeout_secs
+        self.runtime_version = runtime_version or os.environ.get(
+            "FELDERA_RUNTIME_VERSION"
+        )
         if resources is not None:
             self.resources = resources.__dict__
         if isinstance(storage, bool):
@@ -88,6 +93,10 @@ class RuntimeConfig:
         if isinstance(storage, Storage):
             self.storage = storage.__dict__
 
+    @staticmethod
+    def default() -> "RuntimeConfig":
+        return RuntimeConfig(resources=Resources())
+
     @classmethod
     def from_dict(cls, d: Mapping[str, Any]):
         """

{feldera-0.99.0.dist-info → feldera-0.101.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.99.0
+Version: 0.101.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT
@@ -10,9 +10,9 @@ Project-URL: Repository, https://github.com/feldera/feldera
 Project-URL: Issues, https://github.com/feldera/feldera/issues
 Keywords: feldera,python
 Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.10
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.12
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: requests
 Requires-Dist: pandas>=2.1.2
@@ -54,10 +54,11 @@ If you have cloned the Feldera repo, you can install the python SDK as follows:
 pip install python/
 ```
 
-Checkout the docs [here](./feldera/__init__.py) for an example on how to use the SDK.
-
 ## Documentation
 
+The Python SDK documentation is available at
+[Feldera Python SDK Docs](https://docs.feldera.com/python).
+
 To build the html documentation run:
 
 Ensure that you have sphinx installed. If not, install it using `pip install sphinx`.
@@ -77,27 +78,23 @@ To clean the build, run `make clean`.
 To run unit tests:
 
 ```bash
-(cd python && python3 -m unittest)
+cd python && python3 -m pytest tests/
 ```
 
-> ⚠️ Running the unit tests will **delete all existing pipelines**.
-
-The following command runs end-to-end tests.  You'll need a pipeline
-manager running at `http://localhost:8080`.  For the pipeline builder
-tests, you'll also need a broker available at `localhost:9092` and
-(from the pipelines) `redpanda:19092`.  (To change those locations,
-set the environment variables listed in `python/tests/__init__.py`.)
-
-```bash
-(cd python && python3 -m pytest tests)
-```
+- This will detect and run all test files that match the pattern `test_*.py` or
+  `*_test.py`.
+- By default, the tests expect a running Feldera instance at `http://localhost:8080`.
+  To override the default endpoint, set the `FELDERA_BASE_URL` environment variable.
 
 To run tests from a specific file:
 
 ```bash
-(cd python && python3 -m unittest ./tests/path-to-file.py)
+(cd python && python3 -m pytest ./tests/path-to-file.py)
 ```
 
+#### Running Aggregate Tests
+
+The aggregate tests validate end-to-end correctness of SQL functionality.
 To run the aggregate tests use:
 
 ```bash
@@ -105,6 +102,38 @@ cd python
 PYTHONPATH=`pwd` python3 ./tests/aggregate_tests/main.py
 ```
 
+### Reducing Compilation Cycles
+
+To reduce redundant compilation cycles during testing:
+
+* **Inherit from `SharedTestPipeline`** instead of `unittest.TestCase`.
+* **Define DDLs** (e.g., `CREATE TABLE`, `CREATE VIEW`) in the **docstring** of each test method.
+  * All DDLs from all test functions in the class are combined and compiled into a single pipeline.
+  * If a table or view is already defined in one test, it can be used directly in others without redefinition.
+  * Ensure that all table and view names are unique within the class.
+* Use `@enterprise_only` on tests that require Enterprise features. Their DDLs will be skipped on OSS builds.
+* Use `self.set_runtime_config(...)` to override the default pipeline config.
+  * Reset it at the end using `self.reset_runtime_config()`.
+* Access the shared pipeline via `self.pipeline`.
+
+#### Example
+
+```python
+from tests.shared_test_pipeline import SharedTestPipeline
+
+class TestAverage(SharedTestPipeline):
+    def test_average(self):
+        """
+        CREATE TABLE students(id INT, name STRING);
+        CREATE MATERIALIZED VIEW v AS SELECT * FROM students;
+        """
+        ...
+        self.pipeline.start()
+        self.pipeline.input_pandas("students", df)
+        self.pipeline.wait_for_completion(True)
+        ...
+```
+
 ## Linting and formatting
 
 Use [Ruff] to run the lint checks that will be executed by the

feldera-0.101.0.dist-info/RECORD

@@ -0,0 +1,23 @@
+feldera/__init__.py,sha256=EiY3bTj_mnfNhCGrZo6J__brfovIJ-YYAdy77PyaEoo,378
+feldera/_callback_runner.py,sha256=GNOg3TrKJg9zJU0HvpWxCHqzjMUX8ORiHhtiEEdVQzE,4758
+feldera/_helpers.py,sha256=rN0WuGSCCQlXWFMimZUQrgs-LJAfUo074d79sLElncQ,3023
+feldera/enums.py,sha256=AWx7wldY2vBbrJMAwFabGQ8C_ZTKZreiqRvuGlrC9Ks,8702
+feldera/output_handler.py,sha256=64J3ljhOaKIhxdjOKYi-BUz_HnMwROfmN8eE-btYygU,1930
+feldera/pipeline.py,sha256=Wc9yfnEXkVQ_WQSYuENisN6VSzANVJbsSuQGt1K0E_k,34407
+feldera/pipeline_builder.py,sha256=rImmAkYgpCGxZyH317mUMvChLuGF0rH8gvEOVcTYN4M,3924
+feldera/runtime_config.py,sha256=jeVuv2B6H0IDSmkyQgsuSOv1B656jvJIL9re4Sv4Rjw,3781
+feldera/stats.py,sha256=XBhkRsV7FXErwWuPP0i3q9W77mzkMo-oThPVEZy5y3U,5028
+feldera/rest/__init__.py,sha256=Eg-EKUU3RSTDcdxTR_7wNDnCly8VpXEzsZCQUmf-y2M,308
+feldera/rest/_helpers.py,sha256=q7jWInKp9IiIli8N5o31lDG3hNUbcsJqufZXYHG04ps,222
+feldera/rest/_httprequests.py,sha256=e22YbpzOzy7MGo7hk9MOU7ZRTj3F314grY0Ygr-_goI,6636
+feldera/rest/config.py,sha256=DYzZKngDEhouTEwqVFd-rDrBN9tWqsU07Jl_BTT4mXs,1008
+feldera/rest/errors.py,sha256=b4i2JjrbSmej7jdko_FL8UeXklLKenSipwMT80jowaM,1720
+feldera/rest/feldera_client.py,sha256=E0_cqpIaavVeapBLc5egZIl5KVsKWliDzWqXk6_r0nY,25289
+feldera/rest/feldera_config.py,sha256=1pnGbLFMSLvp7Qh_OlPLALSKCSHIktNWKvx6gYU00U4,1374
+feldera/rest/pipeline.py,sha256=-dGGUdtHMABKrQEclaeuwGI_FOCrQOk6p2aCFV0FdU8,2890
+feldera/rest/sql_table.py,sha256=qrw-YwMzx5T81zDefNO1KOx7EyypFz1vPwGBzSUB7kc,652
+feldera/rest/sql_view.py,sha256=hN12mPM0mvwLCIPYywpb12s9Hd2Ws31IlTMXPriMisw,644
+feldera-0.101.0.dist-info/METADATA,sha256=6H0v0BPG56ovszTNICNFycsdWM26sDt3Mdsyp4dQ3_4,4101
+feldera-0.101.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+feldera-0.101.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
+feldera-0.101.0.dist-info/RECORD,,

feldera-0.99.0.dist-info/RECORD

@@ -1,22 +0,0 @@
-feldera/__init__.py,sha256=PxkgCtEAuFwo4u8NGEDio-bF3M-GnbeV45tAQVoBbqE,297
-feldera/_callback_runner.py,sha256=Tdf6BXN4zppyoy8t_y-Ooa3B0wEfvyezMHU9jxY2ZhA,4713
-feldera/_helpers.py,sha256=rN0WuGSCCQlXWFMimZUQrgs-LJAfUo074d79sLElncQ,3023
-feldera/enums.py,sha256=CWPvFIdxMuadS548IAiYjPjBrf3fZe7DXi2VCJGjlhc,8671
-feldera/output_handler.py,sha256=64J3ljhOaKIhxdjOKYi-BUz_HnMwROfmN8eE-btYygU,1930
-feldera/pipeline.py,sha256=OClrsP1Fgg-FjDLCh1doORq6-dQT4lIX1yUDDZtQR0o,34523
-feldera/pipeline_builder.py,sha256=2nu88M6W-a2tBTTwBdUG5Bcqe8y5Gsnvlc5j6BW4OfU,3726
-feldera/runtime_config.py,sha256=EDvnakyaRTHUS8GLd6TqLZ_jsGp7_4fbcPnSfQCw1k0,3501
-feldera/stats.py,sha256=XBhkRsV7FXErwWuPP0i3q9W77mzkMo-oThPVEZy5y3U,5028
-feldera/rest/__init__.py,sha256=Eg-EKUU3RSTDcdxTR_7wNDnCly8VpXEzsZCQUmf-y2M,308
-feldera/rest/_httprequests.py,sha256=e22YbpzOzy7MGo7hk9MOU7ZRTj3F314grY0Ygr-_goI,6636
-feldera/rest/config.py,sha256=DYzZKngDEhouTEwqVFd-rDrBN9tWqsU07Jl_BTT4mXs,1008
-feldera/rest/errors.py,sha256=b4i2JjrbSmej7jdko_FL8UeXklLKenSipwMT80jowaM,1720
-feldera/rest/feldera_client.py,sha256=Vax97r4ZP5ZQ0MsRYFjAxmIXXlCaLNH7EiMN5SfxtJg,25057
-feldera/rest/feldera_config.py,sha256=1pnGbLFMSLvp7Qh_OlPLALSKCSHIktNWKvx6gYU00U4,1374
-feldera/rest/pipeline.py,sha256=-dGGUdtHMABKrQEclaeuwGI_FOCrQOk6p2aCFV0FdU8,2890
-feldera/rest/sql_table.py,sha256=qrw-YwMzx5T81zDefNO1KOx7EyypFz1vPwGBzSUB7kc,652
-feldera/rest/sql_view.py,sha256=hN12mPM0mvwLCIPYywpb12s9Hd2Ws31IlTMXPriMisw,644
-feldera-0.99.0.dist-info/METADATA,sha256=VRmOecA_-shSorbKppGEvg9TI4B002BAF0R0NoxDms8,2892
-feldera-0.99.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-feldera-0.99.0.dist-info/top_level.txt,sha256=fB6yTqrQiO6RCbY1xP2T_mpPoTjDFtJvkJJodiee7d0,8
-feldera-0.99.0.dist-info/RECORD,,