feldera 0.136.0__tar.gz → 0.138.0__tar.gz

{feldera-0.136.0 → feldera-0.138.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.136.0
+Version: 0.138.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

{feldera-0.136.0 → feldera-0.138.0}/feldera/pipeline.py

@@ -326,7 +326,7 @@ class Pipeline:
                     f" time {elapsed}s, timeout: {timeout_s}s"
                 )
 
-            pipeline_complete: bool = self.stats().global_metrics.pipeline_complete
+            pipeline_complete: bool = self.is_complete()
             if pipeline_complete is None:
                 raise RuntimeError(
                     "received unknown metrics from the pipeline, pipeline_complete is None"
@@ -339,6 +339,19 @@ class Pipeline:
         if force_stop:
             self.stop(force=True)
 
+    def is_complete(self) -> bool:
+        """
+        Check if the pipeline has completed processing all input records.
+
+        Returns True if (1) all input connectors attached to the
+        pipeline have finished reading their input data sources and issued
+        end-of-input notifications to the pipeline, and (2) all inputs received
+        from these connectors have been fully processed and corresponding
+        outputs have been sent out through the output connectors.
+        """
+
+        return self.stats().global_metrics.pipeline_complete
+
     def start(self, wait: bool = True, timeout_s: Optional[float] = None):
         """
         .. _start:
@@ -625,6 +638,8 @@ metrics"""
         :param timeout_s: The maximum time (in seconds) to wait for the
             checkpoint to complete.
 
+        :return: The checkpoint sequence number.
+
         :raises FelderaAPIError: If enterprise features are not enabled.
         """
 
@@ -647,9 +662,7 @@ pipeline '{self.name}' to make checkpoint '{seq}'"""
                 time.sleep(0.1)
                 continue
 
-            return status
-
-        return seq
+            return seq
 
     def checkpoint_status(self, seq: int) -> CheckpointStatus:
         """
@@ -889,6 +902,38 @@ pipeline '{self.name}' to sync checkpoint '{uuid}'"""
         self.refresh()
         return self._inner.program_code
 
+    def modify(
+        self,
+        sql: Optional[str] = None,
+        udf_rust: Optional[str] = None,
+        udf_toml: Optional[str] = None,
+        program_config: Optional[Mapping[str, Any]] = None,
+        runtime_config: Optional[Mapping[str, Any]] = None,
+        description: Optional[str] = None,
+    ):
+        """
+        Modify the pipeline.
+
+        Modify the values of pipeline attributes: SQL code, UDF Rust code,
+        UDF Rust dependencies (TOML), program config, runtime config, and
+        description. Only the provided attributes will be modified. Other
+        attributes will remain unchanged.
+
+        The pipeline must be in the STOPPED state to be modified.
+
+        :raises FelderaAPIError: If the pipeline is not in a STOPPED state.
+        """
+
+        self.client.patch_pipeline(
+            name=self._inner.name,
+            sql=sql,
+            udf_rust=udf_rust,
+            udf_toml=udf_toml,
+            program_config=program_config,
+            runtime_config=runtime_config,
+            description=description,
+        )
+
     def storage_status(self) -> StorageStatus:
         """
         Return the storage status of the pipeline.

{feldera-0.136.0 → feldera-0.138.0}/feldera/runtime_config.py

@@ -80,6 +80,7 @@ class RuntimeConfig:
         resources: Optional[Resources] = None,
         fault_tolerance_model: Optional[FaultToleranceModel] = None,
         checkpoint_interval_secs: Optional[int] = None,
+        dev_tweaks: Optional[dict] = None,
     ):
         self.workers = workers
         self.tracing = tracing
@@ -103,6 +104,7 @@ class RuntimeConfig:
                 self.storage = storage.__dict__
             else:
                 raise ValueError(f"Unknown value '{storage}' for storage")
+        self.dev_tweaks = dev_tweaks
 
     @staticmethod
     def default() -> "RuntimeConfig":

feldera-0.138.0/feldera/testutils.py

@@ -0,0 +1,372 @@
+"Utility functions for writing tests against a Feldera instance."
+
+import os
+import re
+import time
+import json
+import unittest
+from typing import List, Optional, cast
+from datetime import datetime
+
+from feldera.enums import CompilationProfile
+from feldera.pipeline import Pipeline
+from feldera.pipeline_builder import PipelineBuilder
+from feldera.runtime_config import Resources, RuntimeConfig
+from feldera.rest import FelderaClient
+
+API_KEY = os.environ.get("FELDERA_API_KEY")
+BASE_URL = (
+    os.environ.get("FELDERA_HOST")
+    or os.environ.get("FELDERA_BASE_URL")
+    or "http://localhost:8080"
+)
+KAFKA_SERVER = os.environ.get("FELDERA_KAFKA_SERVER", "localhost:19092")
+PIPELINE_TO_KAFKA_SERVER = os.environ.get(
+    "FELDERA_PIPELINE_TO_KAFKA_SERVER", "redpanda:9092"
+)
+FELDERA_TLS_INSECURE = True if os.environ.get("FELDERA_TLS_INSECURE") else False
+FELDERA_HTTPS_TLS_CERT = os.environ.get("FELDERA_HTTPS_TLS_CERT")
+if not FELDERA_TLS_INSECURE and FELDERA_HTTPS_TLS_CERT is not None:
+    FELDERA_REQUESTS_VERIFY = FELDERA_HTTPS_TLS_CERT
+else:
+    FELDERA_REQUESTS_VERIFY = not FELDERA_TLS_INSECURE
+
+
+class _LazyClient:
+    "Construct the FelderaClient only when accessed as opposed to when imported."
+
+    __slots__ = ("_client",)
+
+    def __init__(self):
+        self._client = None
+
+    def _ensure(self):
+        if self._client is None:
+            self._client = FelderaClient(
+                connection_timeout=10,
+            )
+        return self._client
+
+    def __getattr__(self, name):
+        return getattr(self._ensure(), name)
+
+    def __call__(self, *a, **kw) -> FelderaClient:
+        return self._ensure()
+
+
+TEST_CLIENT = cast(FelderaClient, _LazyClient())
+
+
+# SQL index definition.
+class IndexSpec:
+    def __init__(self, name: str, columns: List[str]):
+        self.name = name
+        self.columns = columns
+
+    def __repr__(self):
+        return f"IndexSpec(name={self.name!r},columns={self.columns!r})"
+
+
+class ViewSpec:
+    """
+    SQL view definition consisting of a query that can run in Feldera or
+    datafusion, optional connector spec and aux SQL statements, e.g., indexes
+    and lateness clauses following view definition.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        query: str,
+        indexes: List[IndexSpec] = [],
+        connectors: Optional[str] = None,
+        aux: Optional[str] = None,
+        expected_hash: Optional[str] = None,
+    ):
+        if not isinstance(query, str):
+            raise TypeError("query must be a string")
+        self.name = name
+        self.query = query
+        self.connectors = connectors
+        self.indexes = indexes
+        self.aux = aux
+        self.expected_hash = expected_hash
+
+    def __repr__(self):
+        return f"ViewSpec(name={self.name!r}, query={self.query!r}, indexes={self.indexes!r}, connectors={self.connectors!r}, aux={self.aux!r}, expected_hash={self.expected_hash!r})"
+
+    def clone(self):
+        return ViewSpec(
+            self.name,
+            self.query,
+            self.indexes,
+            self.connectors,
+            self.aux,
+            self.expected_hash,
+        )
+
+    def clone_with_name(self, name: str):
+        return ViewSpec(name, self.query, self.indexes, self.connectors, self.aux)
+
+    def sql(self) -> str:
+        sql = ""
+
+        if self.connectors:
+            with_clause = f"\nwith('connectors' = '{self.connectors}')\n"
+        else:
+            with_clause = ""
+
+        sql += (
+            f"create materialized view {self.name}{with_clause} as\n{self.query};\n\n"
+        )
+
+        for index in self.indexes:
+            columns = ",".join(index.columns)
+            sql += f"create index {index.name} on {self.name}({columns});\n"
+
+        if self.aux:
+            sql += f"{self.aux}\n"
+
+        sql += "\n"
+
+        return sql
+
+
+def log(*args, **kwargs):
+    """Print like built-in print(), but prefix each line with current time."""
+    prefix = datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
+    print(prefix, *args, **kwargs)
+
+
+def unique_pipeline_name(base_name: str) -> str:
+    """
+    In CI, multiple tests of different runs can run against the same Feldera instance, we
+    make sure the pipeline names they use are unique by appending the first 5 characters
+    of the commit SHA or 'local' if not in CI.
+    """
+    ci_tag = os.getenv("GITHUB_SHA", "local")[:5]
+    return f"{ci_tag}_{base_name}"
+
+
+def enterprise_only(fn):
+    fn._enterprise_only = True
+    return unittest.skipUnless(
+        TEST_CLIENT.get_config().edition.is_enterprise(),
+        f"{fn.__name__} is enterprise only, skipping",
+    )(fn)
+
+
+def datafusionize(query: str) -> str:
+    sort_array_pattern = re.compile(re.escape("SORT_ARRAY"), re.IGNORECASE)
+    truncate_pattern = re.compile(re.escape("TRUNCATE"), re.IGNORECASE)
+    timestamp_trunc_pattern = re.compile(
+        r"TIMESTAMP_TRUNC\s*\(\s*MAKE_TIMESTAMP\s*\(\s*([^)]+)\s*\)\s*,\s*([A-Z]+)\s*\)",
+        re.IGNORECASE,
+    )
+
+    result = sort_array_pattern.sub("array_sort", query)
+    result = truncate_pattern.sub("trunc", result)
+    result = timestamp_trunc_pattern.sub(r"DATE_TRUNC('\2', TO_TIMESTAMP(\1))", result)
+    return result
+
+
+def validate_view(pipeline: Pipeline, view: ViewSpec):
+    log(f"Validating view '{view.name}'")
+
+    # We have two modes to verify the view, either we run the same SQL as the view against datafusion
+    # by `datafusionizing` the query, or a weaker form where we pass a hash of what the result
+    # should look like and check that the hash hasn't changed
+    if view.expected_hash:
+        view_query = f"select * from {view.name}"
+        computed_hash = pipeline.query_hash(view_query)
+        if computed_hash != view.expected_hash:
+            raise AssertionError(
+                f"View {view.name} hash {computed_hash} was but expected hash {view.expected_hash}"
+            )
+    else:
+        # TODO: count records
+        view_query = datafusionize(view.query)
+        try:
+            extra_rows = list(
+                pipeline.query(f"(select * from {view.name}) except ({view_query})")
+            )
+            missing_rows = list(
+                pipeline.query(f"({view_query}) except (select * from {view.name})")
+            )
+
+            if extra_rows:
+                log("Extra rows in Feldera output, but not in the ad hoc query output")
+                log(json.dumps(extra_rows))
+
+            if missing_rows:
+                log("Extra rows in the ad hoc query output, but not in Feldera output")
+                log(json.dumps(missing_rows))
+        except Exception as e:
+            log(f"Error querying view '{view.name}': {e}")
+            log(f"Ad-hoc Query: {view_query}")
+            raise
+
+        if extra_rows or missing_rows:
+            raise AssertionError(f"Validation failed for view {view.name}")
+
+
+def generate_program(tables: dict, views: List[ViewSpec]) -> str:
+    sql = ""
+
+    for table_sql in tables.values():
+        sql += f"{table_sql}\n"
+
+    for view in views:
+        sql += view.sql()
+
+    return sql
+
+
+def build_pipeline(
+    pipeline_name: str,
+    tables: dict,
+    views: List[ViewSpec],
+    resources: Optional[Resources] = None,
+) -> Pipeline:
+    sql = generate_program(tables, views)
+
+    pipeline = PipelineBuilder(
+        TEST_CLIENT,
+        pipeline_name,
+        sql=sql,
+        compilation_profile=CompilationProfile.OPTIMIZED,
+        runtime_config=RuntimeConfig(
+            provisioning_timeout_secs=60,
+            dev_tweaks={"backfill_avoidance": True},
+            resources=resources,
+        ),
+    ).create_or_replace()
+
+    return pipeline
+
+
+def validate_outputs(pipeline: Pipeline, tables: dict, views: List[ViewSpec]):
+    for table in tables.keys():
+        row_count = list(pipeline.query(f"select count(*) from {table}"))
+        log(f"Table '{table}' count(*):\n{row_count}")
+
+    for view in views:
+        validate_view(pipeline, view)
+
+
+def check_end_of_input(pipeline: Pipeline) -> bool:
+    return all(
+        input_endpoint.metrics.end_of_input
+        for input_endpoint in pipeline.stats().inputs
+    )
+
+
+def wait_end_of_input(pipeline: Pipeline, timeout_s: Optional[int] = None):
+    start_time = time.monotonic()
+    while not check_end_of_input(pipeline):
+        if timeout_s is not None and time.monotonic() - start_time > timeout_s:
+            raise TimeoutError("Timeout waiting for end of input")
+        time.sleep(3)
+
+
+def transaction(pipeline: Pipeline, duration_seconds: int):
+    """Run a transaction for a specified duration."""
+
+    log(f"Running transaction for {duration_seconds} seconds")
+    pipeline.start_transaction()
+    time.sleep(duration_seconds)
+    log("Committing transaction")
+    commit_start = time.monotonic()
+    pipeline.commit_transaction()
+    log(f"Transaction committed in {time.monotonic() - commit_start} seconds")
+
+
+def checkpoint_pipeline(pipeline: Pipeline):
+    """Create a checkpoint and wait for it to complete."""
+
+    log("Creating checkpoint")
+    checkpoint_start = time.monotonic()
+    pipeline.checkpoint(wait=True)
+    log(f"Checkpoint complete in {time.monotonic() - checkpoint_start} seconds")
+
+
+def check_for_endpoint_errors(pipeline: Pipeline):
+    """Check for errors on all input and output endpoints."""
+
+    for input_endpoint_status in pipeline.stats().inputs:
+        input_endpoint_status.metrics
+        if input_endpoint_status.metrics.num_transport_errors > 0:
+            raise RuntimeError(
+                f"Transport errors detected on input endpoint: {input_endpoint_status.endpoint_name}"
+            )
+        if input_endpoint_status.metrics.num_parse_errors > 0:
+            raise RuntimeError(
+                f"Parse errors on input endpoint: {input_endpoint_status.endpoint_name}"
+            )
+        log(f"  Input endpoint {input_endpoint_status.endpoint_name} OK")
+
+    for output_endpoint_status in pipeline.stats().outputs:
+        output_endpoint_status.metrics
+        if output_endpoint_status.metrics.num_transport_errors > 0:
+            raise RuntimeError(
+                f"Transport errors detected on output endpoint: {output_endpoint_status.endpoint_name}"
+            )
+        if output_endpoint_status.metrics.num_encode_errors > 0:
+            raise RuntimeError(
+                f"Encode errors on output endpoint: {output_endpoint_status.endpoint_name}"
+            )
+        log(f"  Output endpoint {output_endpoint_status.endpoint_name} OK")
+
+
+def number_of_processed_records(pipeline: Pipeline) -> int:
+    """Get the total_processed_records metric."""
+
+    return pipeline.stats().global_metrics.total_processed_records
+
+
+def run_workload(
+    pipeline_name: str, tables: dict, views: List[ViewSpec], transaction: bool = True
+):
+    """
+    Helper to run a pipeline to completion and validate the views afterwards using ad-hoc queries.
+
+    Use this for large-scale workload and standard benchmarks (like TPC-H etc.) where you plan to
+    ingest a lot of data and validate the results. For testing more specific functionality, see
+    frameworks in the `tests` directory.
+    """
+
+    pipeline = build_pipeline(pipeline_name, tables, views)
+
+    pipeline.start()
+    start_time = time.monotonic()
+
+    if transaction:
+        try:
+            pipeline.start_transaction()
+        except Exception as e:
+            log(f"Error starting transaction: {e}")
+
+    if transaction:
+        wait_end_of_input(pipeline, timeout_s=3600)
+    else:
+        pipeline.wait_for_completion(force_stop=False, timeout_s=3600)
+
+    elapsed = time.monotonic() - start_time
+    log(f"Data ingested in {elapsed}")
+
+    if transaction:
+        start_time = time.monotonic()
+        try:
+            pipeline.commit_transaction(transaction_id=None, wait=True, timeout_s=None)
+            log(f"Commit took {time.monotonic() - start_time}")
+        except Exception as e:
+            log(f"Error committing transaction: {e}")
+
+        log("Waiting for outputs to flush")
+        start_time = time.monotonic()
+        pipeline.wait_for_completion(force_stop=False, timeout_s=3600)
+        log(f"Flushing outputs took {time.monotonic() - start_time}")
+
+    validate_outputs(pipeline, tables, views)
+
+    pipeline.stop(force=True)

{feldera-0.136.0 → feldera-0.138.0}/feldera.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feldera
-Version: 0.136.0
+Version: 0.138.0
 Summary: The feldera python client
 Author-email: Feldera Team <dev@feldera.com>
 License: MIT

{feldera-0.136.0 → feldera-0.138.0}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "feldera"
 readme = "README.md"
 description = "The feldera python client"
-version = "0.136.0"
+version = "0.138.0"
 license = { text = "MIT" }
 requires-python = ">=3.10"
 authors = [

feldera-0.136.0/feldera/testutils.py

@@ -1,194 +0,0 @@
-"Utility functions for writing tests against a Feldera instance."
-
-import os
-import re
-import time
-import json
-import unittest
-from typing import cast
-
-from feldera.enums import CompilationProfile
-from feldera.pipeline import Pipeline
-from feldera.pipeline_builder import PipelineBuilder
-from feldera.runtime_config import RuntimeConfig
-from feldera.rest import FelderaClient
-
-API_KEY = os.environ.get("FELDERA_API_KEY")
-BASE_URL = (
-    os.environ.get("FELDERA_HOST")
-    or os.environ.get("FELDERA_BASE_URL")
-    or "http://localhost:8080"
-)
-KAFKA_SERVER = os.environ.get("FELDERA_KAFKA_SERVER", "localhost:19092")
-PIPELINE_TO_KAFKA_SERVER = os.environ.get(
-    "FELDERA_PIPELINE_TO_KAFKA_SERVER", "redpanda:9092"
-)
-FELDERA_TLS_INSECURE = True if os.environ.get("FELDERA_TLS_INSECURE") else False
-FELDERA_HTTPS_TLS_CERT = os.environ.get("FELDERA_HTTPS_TLS_CERT")
-if not FELDERA_TLS_INSECURE and FELDERA_HTTPS_TLS_CERT is not None:
-    FELDERA_REQUESTS_VERIFY = FELDERA_HTTPS_TLS_CERT
-else:
-    FELDERA_REQUESTS_VERIFY = not FELDERA_TLS_INSECURE
-
-
-class _LazyClient:
-    "Construct the FelderaClient only when accessed as opposed to when imported."
-
-    __slots__ = ("_client",)
-
-    def __init__(self):
-        self._client = None
-
-    def _ensure(self):
-        if self._client is None:
-            self._client = FelderaClient(
-                connection_timeout=10,
-            )
-        return self._client
-
-    def __getattr__(self, name):
-        return getattr(self._ensure(), name)
-
-    def __call__(self, *a, **kw) -> FelderaClient:
-        return self._ensure()
-
-
-TEST_CLIENT = cast(FelderaClient, _LazyClient())
-
-
-def unique_pipeline_name(base_name: str) -> str:
-    """
-    In CI, multiple tests of different runs can run against the same Feldera instance, we
-    make sure the pipeline names they use are unique by appending the first 5 characters
-    of the commit SHA or 'local' if not in CI.
-    """
-    ci_tag = os.getenv("GITHUB_SHA", "local")[:5]
-    return f"{ci_tag}_{base_name}"
-
-
-def enterprise_only(fn):
-    fn._enterprise_only = True
-    return unittest.skipUnless(
-        TEST_CLIENT.get_config().edition.is_enterprise(),
-        f"{fn.__name__} is enterprise only, skipping",
-    )(fn)
-
-
-def datafusionize(query: str) -> str:
-    sort_array_pattern = re.compile(re.escape("SORT_ARRAY"), re.IGNORECASE)
-    truncate_pattern = re.compile(re.escape("TRUNCATE"), re.IGNORECASE)
-    timestamp_trunc_pattern = re.compile(
-        r"TIMESTAMP_TRUNC\s*\(\s*MAKE_TIMESTAMP\s*\(\s*([^)]+)\s*\)\s*,\s*([A-Z]+)\s*\)",
-        re.IGNORECASE,
-    )
-
-    result = sort_array_pattern.sub("array_sort", query)
-    result = truncate_pattern.sub("trunc", result)
-    result = timestamp_trunc_pattern.sub(r"DATE_TRUNC('\2', TO_TIMESTAMP(\1))", result)
-    return result
-
-
-def validate_view(
-    pipeline: Pipeline, view_name: str, view_query: str | tuple[str, str]
-):
-    print(f"Validating view '{view_name}'")
-
-    # We have two modes to verify the view, either we run the same SQL as the view against datafusion
-    # by `datafusionizing` the query, or a weaker form where we pass a hash of what the result
-    # should look like and check that the hash hasn't changed
-    if isinstance(view_query, tuple):
-        _view_definition, original_hash = view_query
-        view_query = f"select * from {view_name}"
-        computed_hash = pipeline.query_hash(view_query)
-        if computed_hash != original_hash:
-            raise AssertionError(
-                f"View {view_name} hash {computed_hash} was but expected hash {original_hash}"
-            )
-    else:
-        # TODO: count records
-        view_query = datafusionize(view_query)
-        try:
-            extra_rows = list(
-                pipeline.query(f"(select * from {view_name}) except ({view_query})")
-            )
-            missing_rows = list(
-                pipeline.query(f"({view_query}) except (select * from {view_name})")
-            )
-
-            if extra_rows:
-                print(
-                    "Extra rows in Feldera output, but not in the ad hoc query output"
-                )
-                print(json.dumps(extra_rows))
-
-            if missing_rows:
-                print(
-                    "Extra rows in the ad hoc query output, but not in Feldera output"
-                )
-                print(json.dumps(missing_rows))
-        except Exception as e:
-            print(f"Error querying view '{view_name}': {e}")
-            print(f"Ad-hoc Query: {view_query}")
-            raise
-
-        if extra_rows or missing_rows:
-            raise AssertionError(f"Validation failed for view {view_name}")
-
-
-def run_workload(pipeline_name: str, tables: dict, views: dict):
-    """
-    Helper to run a pipeline to completion and validate the views afterwards using ad-hoc queries.
-
-    Use this for large-scale workload and standard benchmarks (like TPC-H etc.) where you plan to
-    ingest a lot of data and validate the results. For testing more specific functionality, see
-    frameworks in the `tests` directory.
-    """
-
-    sql = ""
-    for table_sql in tables.values():
-        sql += f"{table_sql}\n"
-
-    for view_name, view in views.items():
-        if isinstance(view, tuple):
-            view_query, _hash = view
-            sql += f"create materialized view {view_name} as {view_query};\n\n"
-        else:
-            sql += f"create materialized view {view_name} as {view};\n\n"
-
-    pipeline = PipelineBuilder(
-        TEST_CLIENT,
-        unique_pipeline_name(pipeline_name),
-        sql=sql,
-        compilation_profile=CompilationProfile.OPTIMIZED,
-        runtime_config=RuntimeConfig(provisioning_timeout_secs=60),
-    ).create_or_replace()
-
-    pipeline.start()
-    start_time = time.monotonic()
-
-    try:
-        pipeline.start_transaction()
-    except Exception as e:
-        print(f"Error starting transaction: {e}")
-
-    pipeline.wait_for_completion(force_stop=False, timeout_s=3600)
-    elapsed = time.monotonic() - start_time
-    print(f"Data ingested in {elapsed}")
-
-    try:
-        start_time = time.monotonic()
-        pipeline.commit_transaction(transaction_id=None, wait=True, timeout_s=None)
-    except Exception as e:
-        print(f"Error committing transaction: {e}")
-    finally:
-        elapsed = time.monotonic() - start_time
-        print(f"Commit took {elapsed}")
-
-    for table in tables.keys():
-        row_count = list(pipeline.query(f"select count(*) from {table}"))
-        print(f"Table '{table}' count(*):\n{row_count}")
-
-    for view_name, view_query in views.items():
-        validate_view(pipeline, view_name, view_query)
-
-    pipeline.stop(force=True)