feldera 0.69.0__py3-none-any.whl → 0.189.0__py3-none-any.whl

feldera/rest/pipeline.py

@@ -47,15 +47,30 @@ class Pipeline:
         self.program_version: Optional[int] = None
         self.deployment_config: Optional[dict] = None
         self.deployment_desired_status: Optional[str] = None
+        self.deployment_desired_status_since: Optional[str] = None
+        self.deployment_id: Optional[str] = None
+        self.deployment_initial: Optional[str] = None
         self.deployment_error: Optional[dict] = None
         self.deployment_location: Optional[str] = None
-        self.program_binary_url: Optional[str] = None
         self.program_info: Optional[dict] = (
             None  # info about input & output connectors and the schema
         )
         self.program_status: Optional[str] = None
         self.program_status_since: Optional[str] = None
+        self.platform_version: Optional[str] = None
         self.program_error: Optional[dict] = None
+        self.storage_status: Optional[str] = None
+
+        self.deployment_resources_desired_status: Optional[str] = None
+        self.deployment_resources_desired_status_since: Optional[str] = None
+        self.deployment_resources_status: Optional[str] = None
+        self.deployment_resources_status_since: Optional[str] = None
+
+        self.deployment_runtime_desired_status: Optional[str] = None
+        self.deployment_runtime_desired_status_since: Optional[str] = None
+        self.deployment_runtime_status: Optional[str] = None
+        self.deployment_runtime_status_details: Optional[dict] = None
+        self.deployment_runtime_status_since: Optional[str] = None
 
     @classmethod
     def from_dict(cls, d: Mapping[str, Any]):

feldera/runtime_config.py

@@ -1,4 +1,5 @@
 from typing import Optional, Any, Mapping
+from feldera.enums import FaultToleranceModel
 
 
 class Resources:
@@ -58,6 +59,11 @@ class Storage:
 class RuntimeConfig:
     """
     Runtime configuration class to define the configuration for a pipeline.
+    To create runtime config from a dictionary, use
+    :meth:`.RuntimeConfig.from_dict`.
+
+    Documentation:
+        https://docs.feldera.com/pipelines/configuration/#runtime-configuration
     """
 
     def __init__(
@@ -72,6 +78,10 @@ class RuntimeConfig:
         clock_resolution_usecs: Optional[int] = None,
         provisioning_timeout_secs: Optional[int] = None,
         resources: Optional[Resources] = None,
+        fault_tolerance_model: Optional[FaultToleranceModel] = None,
+        checkpoint_interval_secs: Optional[int] = None,
+        dev_tweaks: Optional[dict] = None,
+        logging: Optional[str] = None,
     ):
         self.workers = workers
         self.tracing = tracing
@@ -81,19 +91,36 @@ class RuntimeConfig:
         self.min_batch_size_records = min_batch_size_records
         self.clock_resolution_usecs = clock_resolution_usecs
         self.provisioning_timeout_secs = provisioning_timeout_secs
+        if fault_tolerance_model is not None:
+            self.fault_tolerance = {
+                "model": str(fault_tolerance_model),
+                "checkpoint_interval_secs": checkpoint_interval_secs,
+            }
         if resources is not None:
             self.resources = resources.__dict__
-        if isinstance(storage, bool):
-            self.storage = storage
-        if isinstance(storage, Storage):
-            self.storage = storage.__dict__
+        if storage is not None:
+            if isinstance(storage, bool):
+                self.storage = storage
+            elif isinstance(storage, Storage):
+                self.storage = storage.__dict__
+            else:
+                raise ValueError(f"Unknown value '{storage}' for storage")
+        self.dev_tweaks = dev_tweaks
+        self.logging = logging
+
+    @staticmethod
+    def default() -> "RuntimeConfig":
+        return RuntimeConfig(resources=Resources())
 
     @classmethod
     def from_dict(cls, d: Mapping[str, Any]):
         """
-        Create a `.RuntimeConfig` object from a dictionary.
+        Create a :class:`.RuntimeConfig` object from a dictionary.
         """
 
         conf = cls()
         conf.__dict__ = d
         return conf
+
+    def to_dict(self) -> dict:
+        return dict((k, v) for k, v in self.__dict__.items() if v is not None)

feldera/stats.py

@@ -0,0 +1,152 @@
+from typing import Mapping, Any, Optional, List
+from feldera.enums import PipelineStatus, TransactionStatus
+from datetime import datetime
+import uuid
+
+
+class PipelineStatistics:
+    """
+    Represents statistics reported by a pipeline's "/stats" endpoint.
+    """
+
+    def __init__(self):
+        """
+        Initializes as an empty set of statistics.
+        """
+
+        self.global_metrics: GlobalPipelineMetrics = GlobalPipelineMetrics()
+        self.suspend_error: Optional[Any] = None
+        self.inputs: Mapping[List[InputEndpointStatus]] = {}
+        self.outputs: Mapping[List[OutputEndpointStatus]] = {}
+
+    @classmethod
+    def from_dict(cls, d: Mapping[str, Any]):
+        pipeline = cls()
+        pipeline.global_metrics = GlobalPipelineMetrics.from_dict(d["global_metrics"])
+        pipeline.inputs = [
+            InputEndpointStatus.from_dict(input) for input in d["inputs"]
+        ]
+        pipeline.outputs = [
+            OutputEndpointStatus.from_dict(output) for output in d["outputs"]
+        ]
+        return pipeline
+
+
+class GlobalPipelineMetrics:
+    """Represents the "global_metrics" object within the pipeline's
+    "/stats" endpoint reply.
+    """
+
+    def __init__(self):
+        """
+        Initializes as an empty set of metrics.
+        """
+        self.state: Optional[PipelineStatus] = None
+        self.bootstrap_in_progress: Optional[bool] = None
+        self.rss_bytes: Optional[int] = None
+        self.cpu_msecs: Optional[int] = None
+        self.start_time: Optional[datetime] = None
+        self.incarnation_uuid: Optional[uuid.UUID] = None
+        self.storage_bytes: Optional[int] = None
+        self.storage_mb_secs: Optional[int] = None
+        self.runtime_elapsed_msecs: Optional[int] = None
+        self.buffered_input_records: Optional[int] = None
+        self.total_input_records: Optional[int] = None
+        self.total_processed_records: Optional[int] = None
+        self.total_completed_records: Optional[int] = None
+        self.pipeline_complete: Optional[bool] = None
+        self.transaction_status: Optional[TransactionStatus] = None
+        self.transaction_id: Optional[int] = None
+
+    @classmethod
+    def from_dict(cls, d: Mapping[str, Any]):
+        metrics = cls()
+        metrics.__dict__.update(d)
+        metrics.state = PipelineStatus.from_str(d["state"])
+        metrics.incarnation_uuid = uuid.UUID(d["incarnation_uuid"])
+        metrics.start_time = datetime.fromtimestamp(d["start_time"])
+        metrics.transaction_status = TransactionStatus.from_str(d["transaction_status"])
+        return metrics
+
+
+class InputEndpointStatus:
+    """Represents one member of the "inputs" array within the
+    pipeline's "/stats" endpoint reply.
+    """
+
+    def __init__(self):
+        """Initializes an empty status."""
+        self.endpoint_name: Optional[str] = None
+        self.config: Optional[Mapping] = None
+        self.metrics: Optional[InputEndpointMetrics] = None
+        self.fatal_error: Optional[str] = None
+        self.paused: Optional[bool] = None
+        self.barrier: Optional[bool] = None
+
+    @classmethod
+    def from_dict(cls, d: Mapping[str, Any]):
+        status = cls()
+        status.__dict__.update(d)
+        status.metrics = InputEndpointMetrics.from_dict(d["metrics"])
+        return status
+
+
+class InputEndpointMetrics:
+    """Represents the "metrics" member within an input endpoint status
+    in the pipeline's "/stats" endpoint reply.
+    """
+
+    def __init__(self):
+        self.total_bytes: Optional[int] = None
+        self.total_records: Optional[int] = None
+        self.buffered_records: Optional[int] = None
+        self.num_transport_errors: Optional[int] = None
+        self.num_parse_errors: Optional[int] = None
+        self.end_of_input: Optional[bool] = None
+
+    @classmethod
+    def from_dict(cls, d: Mapping[str, Any]):
+        metrics = cls()
+        metrics.__dict__.update(d)
+        return metrics
+
+
+class OutputEndpointStatus:
+    """Represents one member of the "outputs" array within the
+    pipeline's "/stats" endpoint reply.
+    """
+
+    def __init__(self):
+        """Initializes an empty status."""
+        self.endpoint_name: Optional[str] = None
+        self.config: Optional[Mapping] = None
+        self.metrics: Optional[OutputEndpointMetrics] = None
+        self.fatal_error: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, d: Mapping[str, Any]):
+        status = cls()
+        status.__dict__.update(d)
+        status.metrics = OutputEndpointMetrics.from_dict(d["metrics"])
+        return status
+
+
+class OutputEndpointMetrics:
+    """Represents the "metrics" member within an output endpoint status
+    in the pipeline's "/stats" endpoint reply.
+    """
+
+    def __init__(self):
+        self.transmitted_records: Optional[int] = None
+        self.transmitted_bytes: Optional[int] = None
+        self.queued_records: Optional[int] = None
+        self.queued_batches: Optional[int] = None
+        self.num_encode_errors: Optional[int] = None
+        self.num_transport_errors: Optional[int] = None
+        self.total_processed_input_records: Optional[int] = None
+
+    @classmethod
+    def from_dict(cls, d: Mapping[str, Any]):
+        metrics = cls()
+        metrics.__dict__.update(d)
+        return metrics

feldera/tests/test_datafusionize.py

@@ -0,0 +1,38 @@
+import unittest
+
+from feldera.testutils import datafusionize
+
+
+class TestDatafusionize(unittest.TestCase):
+    def test_datafusionize(self):
+        # Test SORT_ARRAY replacement
+        query = "SELECT SORT_ARRAY(col1) FROM table1"
+        result = datafusionize(query)
+        assert "array_sort(col1)" in result
+
+        # Test TRUNCATE replacement
+        query = "SELECT TRUNCATE(value, 2) FROM table2"
+        result = datafusionize(query)
+        assert "trunc(value, 2)" in result
+
+        # Test TIMESTAMP_TRUNC replacement
+        query = "SELECT TIMESTAMP_TRUNC(MAKE_TIMESTAMP(2023, 1, 15, 10, 30, 0), DAY) FROM table3"
+        result = datafusionize(query)
+        assert "DATE_TRUNC('DAY', TO_TIMESTAMP(2023, 1, 15, 10, 30, 0))" in result
+
+        query = "TIMESTAMP_TRUNC(MAKE_TIMESTAMP(order_group_last_activity_time), hour) AS window_start_time,"
+        result = datafusionize(query)
+        assert (
+            "DATE_TRUNC('hour', TO_TIMESTAMP(order_group_last_activity_time)) AS window_start_time,"
+            in result
+        )
+
+        # Test case insensitive matching
+        query = "SELECT sort_array(col) FROM table WHERE truncate(val) > 0"
+        result = datafusionize(query)
+        assert "array_sort(col)" in result
+        assert "trunc(val)" in result
+
+
+if __name__ == "__main__":
+    unittest.main()

feldera/testutils.py

@@ -0,0 +1,382 @@
+"Utility functions for writing tests against a Feldera instance."
+
+import os
+import re
+import time
+import json
+import unittest
+from typing import List, Optional, cast
+from datetime import datetime
+
+from feldera.enums import CompilationProfile
+from feldera.pipeline import Pipeline
+from feldera.pipeline_builder import PipelineBuilder
+from feldera.runtime_config import Resources, RuntimeConfig
+from feldera.rest import FelderaClient
+from feldera.rest._helpers import requests_verify_from_env
+
+API_KEY = os.environ.get("FELDERA_API_KEY")
+
+
+# OIDC authentication support
+def _get_oidc_token():
+    """Get OIDC token if environment is configured, otherwise return None"""
+    try:
+        from feldera.testutils_oidc import get_oidc_test_helper
+
+        oidc_helper = get_oidc_test_helper()
+        if oidc_helper is not None:
+            return oidc_helper.obtain_access_token()
+    except ImportError:
+        pass
+    return None
+
+
+def _get_effective_api_key():
+    """Get effective API key - OIDC token takes precedence over static API key"""
+    oidc_token = _get_oidc_token()
+    return oidc_token if oidc_token else API_KEY
+
+
+BASE_URL = os.environ.get("FELDERA_HOST") or "http://localhost:8080"
+FELDERA_REQUESTS_VERIFY = requests_verify_from_env()
+
+
+class _LazyClient:
+    "Construct the FelderaClient only when accessed as opposed to when imported."
+
+    __slots__ = ("_client",)
+
+    def __init__(self):
+        self._client = None
+
+    def _ensure(self):
+        if self._client is None:
+            self._client = FelderaClient(
+                connection_timeout=10,
+                api_key=_get_effective_api_key(),
+            )
+        return self._client
+
+    def __getattr__(self, name):
+        return getattr(self._ensure(), name)
+
+    def __call__(self, *a, **kw) -> FelderaClient:
+        return self._ensure()
+
+
+TEST_CLIENT = cast(FelderaClient, _LazyClient())
+
+
+# SQL index definition.
+class IndexSpec:
+    def __init__(self, name: str, columns: List[str]):
+        self.name = name
+        self.columns = columns
+
+    def __repr__(self):
+        return f"IndexSpec(name={self.name!r},columns={self.columns!r})"
+
+
+class ViewSpec:
+    """
+    SQL view definition consisting of a query that can run in Feldera or
+    datafusion, optional connector spec and aux SQL statements, e.g., indexes
+    and lateness clauses following view definition.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        query: str,
+        indexes: List[IndexSpec] = [],
+        connectors: Optional[str] = None,
+        aux: Optional[str] = None,
+        expected_hash: Optional[str] = None,
+    ):
+        if not isinstance(query, str):
+            raise TypeError("query must be a string")
+        self.name = name
+        self.query = query
+        self.connectors = connectors
+        self.indexes = indexes
+        self.aux = aux
+        self.expected_hash = expected_hash
+
+    def __repr__(self):
+        return f"ViewSpec(name={self.name!r}, query={self.query!r}, indexes={self.indexes!r}, connectors={self.connectors!r}, aux={self.aux!r}, expected_hash={self.expected_hash!r})"
+
+    def clone(self):
+        return ViewSpec(
+            self.name,
+            self.query,
+            self.indexes,
+            self.connectors,
+            self.aux,
+            self.expected_hash,
+        )
+
+    def clone_with_name(self, name: str):
+        return ViewSpec(name, self.query, self.indexes, self.connectors, self.aux)
+
+    def sql(self) -> str:
+        sql = ""
+
+        if self.connectors:
+            with_clause = f"\nwith('connectors' = '{self.connectors}')\n"
+        else:
+            with_clause = ""
+
+        sql += (
+            f"create materialized view {self.name}{with_clause} as\n{self.query};\n\n"
+        )
+
+        for index in self.indexes:
+            columns = ",".join(index.columns)
+            sql += f"create index {index.name} on {self.name}({columns});\n"
+
+        if self.aux:
+            sql += f"{self.aux}\n"
+
+        sql += "\n"
+
+        return sql
+
+
+def log(*args, **kwargs):
+    """Print like built-in print(), but prefix each line with current time."""
+    prefix = datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
+    print(prefix, *args, **kwargs)
+
+
+def unique_pipeline_name(base_name: str) -> str:
+    """
+    In CI, multiple tests of different runs can run against the same Feldera instance, we
+    make sure the pipeline names they use are unique by appending the first 5 characters
+    of the commit SHA or 'local' if not in CI.
+    """
+    ci_tag = os.getenv("GITHUB_SHA", "local")[:5]
+    return f"{ci_tag}_{base_name}"
+
+
+def enterprise_only(fn):
+    fn._enterprise_only = True
+    return unittest.skipUnless(
+        TEST_CLIENT.get_config().edition.is_enterprise(),
+        f"{fn.__name__} is enterprise only, skipping",
+    )(fn)
+
+
+def datafusionize(query: str) -> str:
+    sort_array_pattern = re.compile(re.escape("SORT_ARRAY"), re.IGNORECASE)
+    truncate_pattern = re.compile(re.escape("TRUNCATE"), re.IGNORECASE)
+    timestamp_trunc_pattern = re.compile(
+        r"TIMESTAMP_TRUNC\s*\(\s*MAKE_TIMESTAMP\s*\(\s*([^)]+)\s*\)\s*,\s*([A-Z]+)\s*\)",
+        re.IGNORECASE,
+    )
+
+    result = sort_array_pattern.sub("array_sort", query)
+    result = truncate_pattern.sub("trunc", result)
+    result = timestamp_trunc_pattern.sub(r"DATE_TRUNC('\2', TO_TIMESTAMP(\1))", result)
+    return result
+
+
+def validate_view(pipeline: Pipeline, view: ViewSpec):
+    log(f"Validating view '{view.name}'")
+
+    # We have two modes to verify the view, either we run the same SQL as the view against datafusion
+    # by `datafusionizing` the query, or a weaker form where we pass a hash of what the result
+    # should look like and check that the hash hasn't changed
+    if view.expected_hash:
+        view_query = f"select * from {view.name}"
+        computed_hash = pipeline.query_hash(view_query)
+        if computed_hash != view.expected_hash:
+            raise AssertionError(
+                f"View {view.name} hash {computed_hash} was but expected hash {view.expected_hash}"
+            )
+    else:
+        # TODO: count records
+        view_query = datafusionize(view.query)
+        try:
+            extra_rows = list(
+                pipeline.query(f"(select * from {view.name}) except ({view_query})")
+            )
+            missing_rows = list(
+                pipeline.query(f"({view_query}) except (select * from {view.name})")
+            )
+
+            if extra_rows:
+                log("Extra rows in Feldera output, but not in the ad hoc query output")
+                log(json.dumps(extra_rows, default=str))
+
+            if missing_rows:
+                log("Extra rows in the ad hoc query output, but not in Feldera output")
+                log(json.dumps(missing_rows, default=str))
+        except Exception as e:
+            log(f"Error querying view '{view.name}': {e}")
+            log(f"Ad-hoc Query: {view_query}")
+            raise
+
+        if extra_rows or missing_rows:
+            raise AssertionError(f"Validation failed for view {view.name}")
+
+
+def generate_program(tables: dict, views: List[ViewSpec]) -> str:
+    sql = ""
+
+    for table_sql in tables.values():
+        sql += f"{table_sql}\n"
+
+    for view in views:
+        sql += view.sql()
+
+    return sql
+
+
+def build_pipeline(
+    pipeline_name: str,
+    tables: dict,
+    views: List[ViewSpec],
+    resources: Optional[Resources] = None,
+) -> Pipeline:
+    sql = generate_program(tables, views)
+
+    pipeline = PipelineBuilder(
+        TEST_CLIENT,
+        pipeline_name,
+        sql=sql,
+        compilation_profile=CompilationProfile.OPTIMIZED,
+        runtime_config=RuntimeConfig(
+            provisioning_timeout_secs=60,
+            resources=resources,
+        ),
+    ).create_or_replace()
+
+    return pipeline
+
+
+def validate_outputs(pipeline: Pipeline, tables: dict, views: List[ViewSpec]):
+    for table in tables.keys():
+        row_count = list(pipeline.query(f"select count(*) from {table}"))
+        log(f"Table '{table}' count(*):\n{row_count}")
+
+    for view in views:
+        validate_view(pipeline, view)
+
+
+def check_end_of_input(pipeline: Pipeline) -> bool:
+    return all(
+        input_endpoint.metrics.end_of_input
+        for input_endpoint in pipeline.stats().inputs
+    )
+
+
+def wait_end_of_input(pipeline: Pipeline, timeout_s: Optional[int] = None):
+    start_time = time.monotonic()
+    while not check_end_of_input(pipeline):
+        if timeout_s is not None and time.monotonic() - start_time > timeout_s:
+            raise TimeoutError("Timeout waiting for end of input")
+        time.sleep(3)
+
+
+def transaction(pipeline: Pipeline, duration_seconds: int):
+    """Run a transaction for a specified duration."""
+
+    log(f"Running transaction for {duration_seconds} seconds")
+    pipeline.start_transaction()
+    time.sleep(duration_seconds)
+    log("Committing transaction")
+    commit_start = time.monotonic()
+    pipeline.commit_transaction()
+    log(f"Transaction committed in {time.monotonic() - commit_start} seconds")
+
+
+def checkpoint_pipeline(pipeline: Pipeline):
+    """Create a checkpoint and wait for it to complete."""
+
+    log("Creating checkpoint")
+    checkpoint_start = time.monotonic()
+    pipeline.checkpoint(wait=True)
+    log(f"Checkpoint complete in {time.monotonic() - checkpoint_start} seconds")
+
+
+def check_for_endpoint_errors(pipeline: Pipeline):
+    """Check for errors on all input and output endpoints."""
+
+    for input_endpoint_status in pipeline.stats().inputs:
+        input_endpoint_status.metrics
+        if input_endpoint_status.metrics.num_transport_errors > 0:
+            raise RuntimeError(
+                f"Transport errors detected on input endpoint: {input_endpoint_status.endpoint_name}"
+            )
+        if input_endpoint_status.metrics.num_parse_errors > 0:
+            raise RuntimeError(
+                f"Parse errors on input endpoint: {input_endpoint_status.endpoint_name}"
+            )
+        log(f"  Input endpoint {input_endpoint_status.endpoint_name} OK")
+
+    for output_endpoint_status in pipeline.stats().outputs:
+        output_endpoint_status.metrics
+        if output_endpoint_status.metrics.num_transport_errors > 0:
+            raise RuntimeError(
+                f"Transport errors detected on output endpoint: {output_endpoint_status.endpoint_name}"
+            )
+        if output_endpoint_status.metrics.num_encode_errors > 0:
+            raise RuntimeError(
+                f"Encode errors on output endpoint: {output_endpoint_status.endpoint_name}"
+            )
+        log(f"  Output endpoint {output_endpoint_status.endpoint_name} OK")
+
+
+def number_of_processed_records(pipeline: Pipeline) -> int:
+    """Get the total_processed_records metric."""
+
+    return pipeline.stats().global_metrics.total_processed_records
+
+
+def run_workload(
+    pipeline_name: str, tables: dict, views: List[ViewSpec], transaction: bool = True
+):
+    """
+    Helper to run a pipeline to completion and validate the views afterwards using ad-hoc queries.
+
+    Use this for large-scale workload and standard benchmarks (like TPC-H etc.) where you plan to
+    ingest a lot of data and validate the results. For testing more specific functionality, see
+    frameworks in the `tests` directory.
+    """
+
+    pipeline = build_pipeline(pipeline_name, tables, views)
+
+    pipeline.start()
+    start_time = time.monotonic()
+
+    if transaction:
+        try:
+            pipeline.start_transaction()
+        except Exception as e:
+            log(f"Error starting transaction: {e}")
+
+    if transaction:
+        wait_end_of_input(pipeline, timeout_s=3600)
+    else:
+        pipeline.wait_for_completion(force_stop=False, timeout_s=3600)
+
+    elapsed = time.monotonic() - start_time
+    log(f"Data ingested in {elapsed}")
+
+    if transaction:
+        start_time = time.monotonic()
+        try:
+            pipeline.commit_transaction(transaction_id=None, wait=True, timeout_s=None)
+            log(f"Commit took {time.monotonic() - start_time}")
+        except Exception as e:
+            log(f"Error committing transaction: {e}")
+
+        log("Waiting for outputs to flush")
+        start_time = time.monotonic()
+        pipeline.wait_for_completion(force_stop=False, timeout_s=3600)
+        log(f"Flushing outputs took {time.monotonic() - start_time}")
+
+    validate_outputs(pipeline, tables, views)
+
+    pipeline.stop(force=True)