feldera 0.172.0__tar.gz → 0.219.0__tar.gz

feldera-0.219.0/PKG-INFO

@@ -0,0 +1,164 @@
+Metadata-Version: 2.4
+Name: feldera
+Version: 0.219.0
+Summary: The feldera python client
+Author-email: Feldera Team <dev@feldera.com>
+License-Expression: MIT
+Project-URL: Homepage, https://www.feldera.com
+Project-URL: Documentation, https://docs.feldera.com/python
+Project-URL: Repository, https://github.com/feldera/feldera
+Project-URL: Issues, https://github.com/feldera/feldera/issues
+Keywords: feldera,python
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: requests
+Requires-Dist: pandas>=2.1.2
+Requires-Dist: typing-extensions
+Requires-Dist: numpy>=2.2.4
+Requires-Dist: pretty-errors
+Requires-Dist: ruff>=0.6.9
+Requires-Dist: PyJWT>=2.8.0
+
+# Feldera Python SDK
+
+The `feldera` Python package is the Python client for the Feldera HTTP API.
+
+The Python SDK documentation is available at: https://docs.feldera.com/python
+
+## Getting started
+
+### Installation
+
+```bash
+uv pip install feldera
+```
+
+### Example usage
+
+The Python client interacts with the API server of the Feldera instance.
+
+```python
+# File: example.py
+from feldera import FelderaClient, PipelineBuilder, Pipeline
+
+# Instantiate client
+client = FelderaClient()  # Default: http://localhost:8080 without authentication
+# client = FelderaClient(url="https://localhost:8080", api_key="apikey:...", requests_verify="/path/to/tls.crt")
+
+# (Re)create pipeline
+name = "example"
+sql = """
+CREATE TABLE t1 (i1 INT) WITH ('materialized' = 'true');
+CREATE MATERIALIZED VIEW v1 AS SELECT * FROM t1;
+"""
+print("(Re)creating pipeline...")
+pipeline = PipelineBuilder(client, name, sql).create_or_replace()
+pipeline.start()
+print(f"Pipeline status: {pipeline.status()}")
+pipeline.pause()
+print(f"Pipeline status: {pipeline.status()}")
+pipeline.stop(force=True)
+
+# Find existing pipeline
+pipeline = Pipeline.get(name, client)
+pipeline.start()
+print(f"Pipeline status: {pipeline.status()}")
+pipeline.stop(force=True)
+pipeline.clear_storage()
+```
+
+Run using:
+```bash
+uv run python example.py
+```
+
+### Environment variables
+
+Some default parameter values in the Python SDK can be overridden via environment variables.
+
+**Environment variables for `FelderaClient(...)`**
+
+```bash
+export FELDERA_HOST="https://localhost:8080"  # Overrides default for `url`
+export FELDERA_API_KEY="apikey:..."  # Overrides default for `api_key`
+
+# The following together override default for `requests_verify`
+# export FELDERA_TLS_INSECURE="false"  # If set to "1", "true" or "yes" (all case-insensitive), disables TLS certificate verification
+# export FELDERA_HTTPS_TLS_CERT="/path/to/tls.crt"  # Custom TLS certificate
+```
+
+**Environment variables for `PipelineBuilder(...)`**
+
+```bash
+export FELDERA_RUNTIME_VERSION="..."  # Overrides default for `runtime_version`
+```
+
+## Development
+
+Development assumes you have cloned the Feldera code repository.
+
+### Installation
+
+```bash
+cd python
+# Optional: create and activate virtual environment if you don't have one
+uv venv
+source .venv/bin/activate
+# Install in editable mode
+uv pip install -e .
+```
+
+### Formatting
+
+Formatting requires the `ruff` package: `uv pip install ruff`
+
+```bash
+cd python
+ruff check
+ruff format
+```
+
+### Tests
+
+Running the test requires the `pytest` package: `uv pip install pytest`
+
+```bash
+# All tests
+cd python
+uv run python -m pytest tests/
+
+# Specific tests directory
+uv run python -m pytest tests/platform/
+
+# Specific test file
+uv run python -m pytest tests/platform/test_pipeline_crud.py
+
+# Tip: add argument -x at the end for it to fail fast
+```
+
+For further information about the tests, please see `tests/README.md`.
+
+### Documentation
+
+Building documentation requires the `sphinx` package: `uv pip install sphinx`
+
+```bash
+cd python/docs
+sphinx-apidoc -o . ../feldera
+make html
+make clean  # Cleanup afterwards
+```
+
+### Installation from GitHub
+
+Latest `main` branch:
+```bash
+uv pip install git+https://github.com/feldera/feldera#subdirectory=python
+```
+
+Different branch (replace `BRANCH_NAME`):
+```bash
+uv pip install git+https://github.com/feldera/feldera@BRANCH_NAME#subdirectory=python
+```

feldera-0.219.0/README.md

@@ -0,0 +1,141 @@
+# Feldera Python SDK
+
+The `feldera` Python package is the Python client for the Feldera HTTP API.
+
+The Python SDK documentation is available at: https://docs.feldera.com/python
+
+## Getting started
+
+### Installation
+
+```bash
+uv pip install feldera
+```
+
+### Example usage
+
+The Python client interacts with the API server of the Feldera instance.
+
+```python
+# File: example.py
+from feldera import FelderaClient, PipelineBuilder, Pipeline
+
+# Instantiate client
+client = FelderaClient()  # Default: http://localhost:8080 without authentication
+# client = FelderaClient(url="https://localhost:8080", api_key="apikey:...", requests_verify="/path/to/tls.crt")
+
+# (Re)create pipeline
+name = "example"
+sql = """
+CREATE TABLE t1 (i1 INT) WITH ('materialized' = 'true');
+CREATE MATERIALIZED VIEW v1 AS SELECT * FROM t1;
+"""
+print("(Re)creating pipeline...")
+pipeline = PipelineBuilder(client, name, sql).create_or_replace()
+pipeline.start()
+print(f"Pipeline status: {pipeline.status()}")
+pipeline.pause()
+print(f"Pipeline status: {pipeline.status()}")
+pipeline.stop(force=True)
+
+# Find existing pipeline
+pipeline = Pipeline.get(name, client)
+pipeline.start()
+print(f"Pipeline status: {pipeline.status()}")
+pipeline.stop(force=True)
+pipeline.clear_storage()
+```
+
+Run using:
+```bash
+uv run python example.py
+```
+
+### Environment variables
+
+Some default parameter values in the Python SDK can be overridden via environment variables.
+
+**Environment variables for `FelderaClient(...)`**
+
+```bash
+export FELDERA_HOST="https://localhost:8080"  # Overrides default for `url`
+export FELDERA_API_KEY="apikey:..."  # Overrides default for `api_key`
+
+# The following together override default for `requests_verify`
+# export FELDERA_TLS_INSECURE="false"  # If set to "1", "true" or "yes" (all case-insensitive), disables TLS certificate verification
+# export FELDERA_HTTPS_TLS_CERT="/path/to/tls.crt"  # Custom TLS certificate
+```
+
+**Environment variables for `PipelineBuilder(...)`**
+
+```bash
+export FELDERA_RUNTIME_VERSION="..."  # Overrides default for `runtime_version`
+```
+
+## Development
+
+Development assumes you have cloned the Feldera code repository.
+
+### Installation
+
+```bash
+cd python
+# Optional: create and activate virtual environment if you don't have one
+uv venv
+source .venv/bin/activate
+# Install in editable mode
+uv pip install -e .
+```
+
+### Formatting
+
+Formatting requires the `ruff` package: `uv pip install ruff`
+
+```bash
+cd python
+ruff check
+ruff format
+```
+
+### Tests
+
+Running the test requires the `pytest` package: `uv pip install pytest`
+
+```bash
+# All tests
+cd python
+uv run python -m pytest tests/
+
+# Specific tests directory
+uv run python -m pytest tests/platform/
+
+# Specific test file
+uv run python -m pytest tests/platform/test_pipeline_crud.py
+
+# Tip: add argument -x at the end for it to fail fast
+```
+
+For further information about the tests, please see `tests/README.md`.
+
+### Documentation
+
+Building documentation requires the `sphinx` package: `uv pip install sphinx`
+
+```bash
+cd python/docs
+sphinx-apidoc -o . ../feldera
+make html
+make clean  # Cleanup afterwards
+```
+
+### Installation from GitHub
+
+Latest `main` branch:
+```bash
+uv pip install git+https://github.com/feldera/feldera#subdirectory=python
+```
+
+Different branch (replace `BRANCH_NAME`):
+```bash
+uv pip install git+https://github.com/feldera/feldera@BRANCH_NAME#subdirectory=python
+```

{feldera-0.172.0 → feldera-0.219.0}/feldera/__init__.py

@@ -1,9 +1,9 @@
 from feldera.rest.feldera_client import FelderaClient as FelderaClient
 from feldera.pipeline import Pipeline as Pipeline
 from feldera.pipeline_builder import PipelineBuilder as PipelineBuilder
-from feldera.rest._helpers import client_version
+from feldera.rest._helpers import determine_client_version
 
-__version__ = client_version()
+__version__ = determine_client_version()
 
 import pretty_errors
 

{feldera-0.172.0 → feldera-0.219.0}/feldera/pipeline.py

@@ -1,36 +1,35 @@
 import logging
+import pathlib
 import time
+from collections import deque
 from datetime import datetime
-import pathlib
-
-import pandas
+from threading import Event
+from typing import Any, Callable, Dict, Generator, List, Mapping, Optional
 from uuid import UUID
 
-from typing import List, Dict, Callable, Optional, Generator, Mapping, Any
-from threading import Event
-from collections import deque
+import pandas
 
-from feldera.rest.errors import FelderaAPIError
+from feldera._callback_runner import CallbackRunner
+from feldera._helpers import chunk_dataframe, ensure_dataframe_has_columns
 from feldera.enums import (
     BootstrapPolicy,
-    CompletionTokenStatus,
-    PipelineFieldSelector,
-    PipelineStatus,
-    ProgramStatus,
     CheckpointStatus,
-    TransactionStatus,
-    StorageStatus,
+    CompletionTokenStatus,
     DeploymentDesiredStatus,
     DeploymentResourcesDesiredStatus,
     DeploymentResourcesStatus,
     DeploymentRuntimeDesiredStatus,
     DeploymentRuntimeStatus,
+    PipelineFieldSelector,
+    PipelineStatus,
+    ProgramStatus,
+    StorageStatus,
+    TransactionStatus,
 )
-from feldera.rest.pipeline import Pipeline as InnerPipeline
-from feldera.rest.feldera_client import FelderaClient
-from feldera._callback_runner import CallbackRunner
 from feldera.output_handler import OutputHandler
-from feldera._helpers import ensure_dataframe_has_columns, chunk_dataframe
+from feldera.rest.errors import FelderaAPIError
+from feldera.rest.feldera_client import FelderaClient
+from feldera.rest.pipeline import Pipeline as InnerPipeline
 from feldera.rest.sql_table import SQLTable
 from feldera.rest.sql_view import SQLView
 from feldera.runtime_config import RuntimeConfig
@@ -826,10 +825,11 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
 
         resp = self.client.sync_checkpoint_status(self.name)
         success = resp.get("success")
+        periodic = resp.get("periodic")
 
         fail = resp.get("failure") or {}
 
-        if uuid == success:
+        if uuid == success or uuid == periodic:
             return CheckpointStatus.Success
 
         fail = resp.get("failure") or {}
@@ -844,6 +844,26 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
 
         return CheckpointStatus.Unknown
 
+    def last_successful_checkpoint_sync(self) -> UUID:
+        """
+        Returns the UUID of the last successfully synced checkpoint.
+
+        :return: The UUID of the last successfully synced checkpoint.
+        """
+
+        resp = self.client.sync_checkpoint_status(self.name)
+        success = resp.get("success")
+        periodic = resp.get("periodic")
+
+        if success is None and periodic is None:
+            raise RuntimeError("no checkpoints have been synced yet")
+        elif success is None:
+            return UUID(periodic)
+        elif periodic is None:
+            return UUID(success)
+        else:
+            return max(UUID(success), UUID(periodic))
+
     def query(self, query: str) -> Generator[Mapping[str, Any], None, None]:
         """
         Executes an ad-hoc SQL query on this pipeline and returns a generator
@@ -1337,6 +1357,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         stats: bool = True,
         pipeline_config: bool = True,
         system_config: bool = True,
+        dataflow_graph: bool = True,
     ) -> bytes:
         """
         Generate a support bundle containing diagnostic information from this pipeline.
@@ -1354,6 +1375,7 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         :param stats: Whether to collect stats data (default: True)
         :param pipeline_config: Whether to collect pipeline configuration data (default: True)
         :param system_config: Whether to collect system configuration data (default: True)
+        :param dataflow_graph: Whether to collect dataflow graph (default: True)
         :return: The support bundle as bytes (ZIP archive)
         :raises FelderaAPIError: If the pipeline does not exist or if there's an error
         """
@@ -1374,6 +1396,8 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
             params["pipeline_config"] = "false"
         if not system_config:
             params["system_config"] = "false"
+        if not dataflow_graph:
+            params["dataflow_graph"] = "false"
 
         support_bundle_bytes = self.client.get_pipeline_support_bundle(
             self.name, params=params
@@ -1393,6 +1417,42 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
 
         return support_bundle_bytes
 
+    def start_samply_profile(self, duration: int = 30):
+        """
+        Starts profiling this pipeline using samply.
+
+        :param duration: The duration of the profile in seconds (default: 30)
+        :raises FelderaAPIError: If the pipeline does not exist or if there's an error
+        """
+
+        self.client.start_samply_profile(self.name, duration)
+
+    def get_samply_profile(self, output_path: Optional[str] = None) -> bytes:
+        """
+        Returns the gzip file of the samply profile as bytes.
+
+        The gzip file contains the samply profile that can be inspected by the samply tool.
+
+        :param output_path: Optional path to save the samply profile file. If None,
+            the samply profile is only returned as bytes.
+        :return: The samply profile as bytes (GZIP file)
+        :raises FelderaAPIError: If the pipeline does not exist or if there's an error
+        """
+
+        samply_profile_bytes = self.client.get_samply_profile(self.name)
+
+        if output_path is not None:
+            path = pathlib.Path(output_path)
+
+            if path.suffix != ".gz":
+                path = path.with_suffix(".gz")
+            with open(path, "wb") as f:
+                f.write(samply_profile_bytes)
+
+            print(f"Samply profile written to {path}")
+
+        return samply_profile_bytes
+
     def generate_completion_token(self, table_name: str, connector_name: str) -> str:
         """
         Returns a completion token that can be passed to :meth:`.Pipeline.completion_token_status` to

{feldera-0.172.0 → feldera-0.219.0}/feldera/pipeline_builder.py

@@ -1,12 +1,12 @@
 import os
 from typing import Optional
 
+from feldera.enums import CompilationProfile, PipelineFieldSelector
+from feldera.pipeline import Pipeline
+from feldera.rest.errors import FelderaAPIError
 from feldera.rest.feldera_client import FelderaClient
 from feldera.rest.pipeline import Pipeline as InnerPipeline
-from feldera.pipeline import Pipeline
-from feldera.enums import CompilationProfile, PipelineFieldSelector
 from feldera.runtime_config import RuntimeConfig
-from feldera.rest.errors import FelderaAPIError
 
 
 class PipelineBuilder:
@@ -49,10 +49,11 @@ class PipelineBuilder:
             "FELDERA_RUNTIME_VERSION", runtime_version
         )
 
-    def create(self) -> Pipeline:
+    def create(self, wait: bool = True) -> Pipeline:
         """
         Create the pipeline if it does not exist.
 
+        :param wait: Whether to wait for the pipeline to be created. True by default
         :return: The created pipeline
         """
 
@@ -82,17 +83,20 @@ class PipelineBuilder:
             runtime_config=self.runtime_config.to_dict(),
         )
 
-        inner = self.client.create_pipeline(inner)
+        inner = self.client.create_pipeline(inner, wait=wait)
         pipeline = Pipeline(self.client)
         pipeline._inner = inner
 
         return pipeline
 
-    def create_or_replace(self) -> Pipeline:
+    def create_or_replace(self, wait: bool = True) -> Pipeline:
         """
         Creates a pipeline if it does not exist and replaces it if it exists.
 
         If the pipeline exists and is running, it will be stopped and replaced.
+
+        :param wait: Whether to wait for the pipeline to be created. True by default
+        :return: The created pipeline
         """
 
         if self.name is None or self.sql is None:
@@ -121,7 +125,7 @@ class PipelineBuilder:
             runtime_config=self.runtime_config.to_dict(),
         )
 
-        inner = self.client.create_or_update_pipeline(inner)
+        inner = self.client.create_or_update_pipeline(inner, wait=wait)
         pipeline = Pipeline(self.client)
         pipeline._inner = inner
 

{feldera-0.172.0 → feldera-0.219.0}/feldera/rest/_helpers.py

@@ -2,7 +2,7 @@ import logging
 import os
 
 
-def client_version() -> str:
+def determine_client_version() -> str:
     from importlib.metadata import version, PackageNotFoundError
 
     try: