streamsets 6.6.1rc1__tar.gz → 6.6.2.dev0__tar.gz

{streamsets-6.6.1rc1 → streamsets-6.6.2.dev0}/HISTORY.rst

@@ -3,6 +3,12 @@
 
 History
 =======
+6.6.1 (June 2025)
+----------------------
+* Fixed an issue that caused an error to be raised when calling the method :py:meth:`streamsets.sdk.ControlHub.delete_deployment`.
+
+* Bug fixes and improvements
+
 6.6.0 (June 2025)
 -----------------
 * Added support for interacting with network access rules

{streamsets-6.6.1rc1/streamsets.egg-info → streamsets-6.6.2.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: streamsets
-Version: 6.6.1rc1
+Version: 6.6.2.dev0
 Summary: A Python SDK for StreamSets
 Author: StreamSets Inc
 Classifier: Development Status :: 5 - Production/Stable

streamsets-6.6.2.dev0/streamsets/sdk/__version__.py

@@ -0,0 +1,5 @@
+#  IBM Confidential
+#  PID 5900-BAF
+#  Copyright StreamSets Inc., an IBM Company 2025
+
+__version__ = "6.6.2dev0"

{streamsets-6.6.1rc1 → streamsets-6.6.2.dev0}/streamsets/sdk/sch_models.py

@@ -3632,6 +3632,150 @@ class Pipeline(BaseModel):
 
         return stage_list
 
+    @property
+    def start_event(self):
+        """Get the Start Event of the Pipeline."""
+        library_definitions = self._library_definitions
+        stage = self._pipeline_definition.get('startEventStages', [{}])[0]
+
+        if not stage:
+            return None
+
+        stage_definition = next(
+            (stage_def for stage_def in library_definitions['stages'] if stage_def['name'] == stage['stageName']),
+            {},
+        )
+
+        stage_class = self._get_stage_class(stage_definition, library_definitions)
+        stage_object = stage_class(stage=stage, pipeline=self)
+
+        return stage_object
+
+    @property
+    def stop_event(self):
+        """Get the Stop Event of the Pipeline."""
+        library_definitions = self._library_definitions
+        stage = self._pipeline_definition.get('stopEventStages', [{}])[0]
+
+        if not stage:
+            return None
+
+        stage_definition = next(
+            (stage_def for stage_def in library_definitions['stages'] if stage_def['name'] == stage['stageName']),
+            {},
+        )
+
+        stage_class = self._get_stage_class(stage_definition, library_definitions)
+        stage_object = stage_class(stage=stage, pipeline=self)
+
+        return stage_object
+
+    @property
+    def test_origin(self):
+        """Get the Test Origin of the Pipeline."""
+        library_definitions = self._library_definitions
+        stage = self._pipeline_definition.get('testOriginStage', [{}])
+
+        if not stage:
+            return None
+
+        stage_definition = next(
+            (stage_def for stage_def in library_definitions['stages'] if stage_def['name'] == stage['stageName']),
+            {},
+        )
+
+        stage_class = self._get_stage_class(stage_definition, library_definitions)
+        stage_object = stage_class(stage=stage, pipeline=self)
+
+        return stage_object
+
+    def set_start_event(self, label=None, name=None, library=None):
+        """Set the Start Event of the Pipeline."""
+
+        # If _library_definitions is not populated, we call the internal _library_definitions property to perform
+        # lazy-loading of the underlying data
+        if not self._data_internal['libraryDefinitions']:
+            self._library_definitions
+
+        try:
+            builder = self._get_builder()
+        # ValueError gets raised by the Engines class when trying to find an inaccessible Engine
+        # However, sometimes ControlHub finds the Inaccessible Engine instance but we then error out later
+        # downstream of the get_pipeline_builder method and throws an InternalServerError
+        except (ValueError, InternalServerError) as e:
+            raise ValueError(
+                'Editing pipeline {} is not supported when the engine is not accessible,'
+                ' please check if the engine is running and try again.'.format(self.pipeline_id)
+            ) from e
+
+        stage = builder.add_start_event_stage(label=label, name=name, library=library)
+
+        # Pass the newly created stage into the Pipeline
+        self._pipeline_definition['startEventStages'] = builder._pipeline[builder._config_key]['startEventStages']
+
+        # Set the value in the Pipeline configuration
+        self.configuration.start_event = self._builder._stage_to_configuration_name(stage._data)
+
+        return stage
+
+    def set_stop_event(self, label=None, name=None, library=None):
+        """Set the Stop Event of the Pipeline."""
+
+        # If _library_definitions is not populated, we call the internal _library_definitions property to perform
+        # lazy-loading of the underlying data
+        if not self._data_internal['libraryDefinitions']:
+            self._library_definitions
+
+        try:
+            builder = self._get_builder()
+        # ValueError gets raised by the Engines class when trying to find an inaccessible Engine
+        # However, sometimes ControlHub finds the Inaccessible Engine instance but we then error out later
+        # downstream of the get_pipeline_builder method and throws an InternalServerError
+        except (ValueError, InternalServerError) as e:
+            raise ValueError(
+                'Editing pipeline {} is not supported when the engine is not accessible,'
+                ' please check if the engine is running and try again.'.format(self.pipeline_id)
+            ) from e
+
+        stage = builder.add_stop_event_stage(label=label, name=name, library=library)
+
+        # Pass the newly created stage into the Pipeline
+        self._pipeline_definition['stopEventStages'] = builder._pipeline[builder._config_key]['stopEventStages']
+
+        # Set the value in the Pipeline configuration
+        self.configuration.stop_event = self._builder._stage_to_configuration_name(stage._data)
+
+        return stage
+
+    def set_test_origin(self, label=None, name=None, library=None):
+        """Set the Test Origin of the Pipeline."""
+
+        # If _library_definitions is not populated, we call the internal _library_definitions property to perform
+        # lazy-loading of the underlying data
+        if not self._data_internal['libraryDefinitions']:
+            self._library_definitions
+
+        try:
+            builder = self._get_builder()
+        # ValueError gets raised by the Engines class when trying to find an inaccessible Engine
+        # However, sometimes ControlHub finds the Inaccessible Engine instance but we then error out later
+        # downstream of the get_pipeline_builder method and throws an InternalServerError
+        except (ValueError, InternalServerError) as e:
+            raise ValueError(
+                'Editing pipeline {} is not supported when the engine is not accessible,'
+                ' please check if the engine is running and try again.'.format(self.pipeline_id)
+            ) from e
+
+        stage = builder.add_test_origin_stage(label=label, name=name, library=library)
+
+        # Pass the newly created stage into the Pipeline
+        self._pipeline_definition['testOriginStage'] = builder._pipeline[builder._config_key]['testOriginStage']
+
+        # Set the value in Pipeline configuration
+        self.configuration.test_origin = self._builder._stage_to_configuration_name(stage._data)
+
+        return stage
+
     @property
     def error_stage(self):
         """Get the error stage of the Pipeline."""

{streamsets-6.6.1rc1 → streamsets-6.6.2.dev0}/streamsets/sdk/sdc_models.py

@@ -21,8 +21,8 @@ from .constants import (
 from .exceptions import ServiceDefinitionNotFound
 from .models import Configuration, _StageWithPredicates
 from .utils import (
-    SeekableList, determine_fragment_label, format_log, get_attribute, get_color_icon_from_stage_definition,
-    get_open_output_lanes, get_params, pipeline_json_encoder,
+    SeekableList, determine_fragment_label, format_log, get_accepted_labels_libraries_and_names, get_attribute,
+    get_color_icon_from_stage_definition, get_open_output_lanes, get_params, pipeline_json_encoder,
 )
 
 # fmt: on
@@ -1062,11 +1062,28 @@ class PipelineBuilder:
         Returns:
             An instance of :py:class:`streamsets.sdk.sdc_models.Stage`.
         """
+        accepted_labels, accepted_libraries, accepted_java_names = get_accepted_labels_libraries_and_names(
+            self, 'startEventStage'
+        )
+
+        if label and label not in accepted_labels:
+            raise ValueError('Stage with label {} is not a supported start event stage'.format(label))
+        if name and name not in accepted_java_names:
+            raise ValueError('Stage with name {} is not a supported start event stage'.format(name))
+        if library and library not in accepted_libraries:
+            raise ValueError('Stage with library {} is not a supported start event stage'.format(library))
+
         stage_instance, stage_definition = next(
             (stage.instance, stage.definition)
             for stage in self._get_stage_data(label=label, name=name, library=library)
             if stage.definition.get('pipelineLifecycleStage') is True
         )
+
+        # We need instanceName in the form <instance_name>_StartEventStage and not <instance_name>_01
+        instance_name_split_list = stage_instance['instanceName'].split('_')
+        instance_name_split_list[-1] = 'StartEventStage'
+        stage_instance['instanceName'] = '_'.join(instance_name_split_list)
+
         self._pipeline[self._config_key]['startEventStages'] = [stage_instance]
         self._set_pipeline_configuration('startEventStage', self._stage_to_configuration_name(stage_instance))
         return self._all_stages.get(stage_instance['stageName'], Stage)(
@@ -1092,11 +1109,28 @@ class PipelineBuilder:
         Returns:
             An instance of :py:class:`streamsets.sdk.sdc_models.Stage`.
         """
+        accepted_labels, accepted_libraries, accepted_java_names = get_accepted_labels_libraries_and_names(
+            self, 'stopEventStage'
+        )
+
+        if label and label not in accepted_labels:
+            raise ValueError('Stage with label {} is not a supported stop event stage'.format(label))
+        if name and name not in accepted_java_names:
+            raise ValueError('Stage with name {} is not a supported stop event stage'.format(name))
+        if library and library not in accepted_libraries:
+            raise ValueError('Stage with library {} is not a supported stop event stage'.format(library))
+
         stage_instance, stage_definition = next(
             (stage.instance, stage.definition)
             for stage in self._get_stage_data(label=label, name=name, library=library)
             if stage.definition.get('pipelineLifecycleStage') is True
         )
+
+        # We need instanceName in the form <instance_name>_StartEventStage and not <instance_name>_01
+        instance_name_split_list = stage_instance['instanceName'].split('_')
+        instance_name_split_list[-1] = 'StopEventStage'
+        stage_instance['instanceName'] = '_'.join(instance_name_split_list)
+
         self._pipeline[self._config_key]['stopEventStages'] = [stage_instance]
         self._set_pipeline_configuration('stopEventStage', self._stage_to_configuration_name(stage_instance))
         return self._all_stages.get(stage_instance['stageName'], Stage)(
@@ -1121,12 +1155,30 @@ class PipelineBuilder:
 
         Returns:
             An instance of :py:class:`streamsets.sdk.sdc_models.Stage`.
+
         """
+        accepted_labels, accepted_libraries, accepted_java_names = get_accepted_labels_libraries_and_names(
+            self, 'testOriginStage'
+        )
+
+        if label and label not in accepted_labels:
+            raise ValueError('Stage with label {} is not a supported test origin stage'.format(label))
+        if name and name not in accepted_java_names:
+            raise ValueError('Stage with name {} is not a supported test origin stage'.format(name))
+        if library and library not in accepted_libraries:
+            raise ValueError('Stage with library {} is not a supported test origin stage'.format(library))
+
         stage_instance, stage_definition = next(
             (stage.instance, stage.definition)
             for stage in self._get_stage_data(label=label, name=name, library=library)
             if stage.definition.get('type') == 'SOURCE'
         )
+
+        # We need instanceName in the form <instance_name>_StartEventStage and not <instance_name>_01
+        instance_name_split_list = stage_instance['instanceName'].split('_')
+        instance_name_split_list[-1] = 'TestOriginStage'
+        stage_instance['instanceName'] = '_'.join(instance_name_split_list)
+
         self._pipeline[self._config_key]['testOriginStage'] = stage_instance
         self._set_pipeline_configuration('testOriginStage', self._stage_to_configuration_name(stage_instance))
         return self._all_stages.get(stage_instance['stageName'], Stage)(

{streamsets-6.6.1rc1 → streamsets-6.6.2.dev0}/streamsets/sdk/st_models.py

@@ -19,7 +19,8 @@ from .constants import ST_STAGE_ATTRIBUTE_RENAME_ALIASES as STAGE_ATTRIBUTE_RENA
 from .constants import ServiceConfigurationProperty, StageConfigurationProperty
 from .models import Configuration, _StageWithPredicates
 from .utils import (
-    SeekableList, format_log, get_attribute, get_color_icon_from_stage_definition, get_params, pipeline_json_encoder,
+    SeekableList, format_log, get_accepted_labels_libraries_and_names, get_attribute,
+    get_color_icon_from_stage_definition, get_params, pipeline_json_encoder,
 )
 
 # fmt: on
@@ -897,14 +898,30 @@ class PipelineBuilder:
         Returns:
             An instance of :py:class:`streamsets.sdk.st_models.Stage`.
         """
+        accepted_labels, accepted_libraries, accepted_java_names = get_accepted_labels_libraries_and_names(
+            self, 'startEventStage'
+        )
+
+        if label and label not in accepted_labels:
+            raise ValueError('Stage with label {} is not a supported start event stage'.format(label))
+        if name and name not in accepted_java_names:
+            raise ValueError('Stage with name {} is not a supported start event stage'.format(name))
+        if library and library not in accepted_libraries:
+            raise ValueError('Stage with library {} is not a supported start event stage'.format(library))
+
         stage_instance, stage_definition = next(
             (stage.instance, stage.definition)
             for stage in self._get_stage_data(label=label, name=name, library=library)
             if stage.definition.get('pipelineLifecycleStage') is True
         )
-        self._pipeline['pipelineConfig']['startEventStages'] = [stage_instance]
-        self._set_pipeline_configuration('startEventStage', self._stage_to_configuration_name(stage_instance))
 
+        # We need instanceName in the form <instance_name>_StartEventStage and not <instance_name>_01
+        instance_name_split_list = stage_instance['instanceName'].split('_')
+        instance_name_split_list[-1] = 'StartEventStage'
+        stage_instance['instanceName'] = '_'.join(instance_name_split_list)
+
+        self._pipeline[self._config_key]['startEventStages'] = [stage_instance]
+        self._set_pipeline_configuration('startEventStage', self._stage_to_configuration_name(stage_instance))
         return self._all_stages.get(stage_instance['stageName'], Stage)(
             stage=stage_instance,
             output_streams=stage_definition.get('outputStreams', 0),
@@ -928,14 +945,31 @@ class PipelineBuilder:
         Returns:
             An instance of :py:class:`streamsets.sdk.st_models.Stage`.
         """
+        # Create accepted lists
+        accepted_labels, accepted_libraries, accepted_java_names = get_accepted_labels_libraries_and_names(
+            self, 'stopEventStage'
+        )
+
+        if label and label not in accepted_labels:
+            raise ValueError('Stage with label {} is not a supported stop event stage'.format(label))
+        if name and name not in accepted_java_names:
+            raise ValueError('Stage with name {} is not a supported stop event stage'.format(name))
+        if library and library not in accepted_libraries:
+            raise ValueError('Stage with library {} is not a supported stop event stage'.format(library))
+
         stage_instance, stage_definition = next(
             (stage.instance, stage.definition)
             for stage in self._get_stage_data(label=label, name=name, library=library)
             if stage.definition.get('pipelineLifecycleStage') is True
         )
-        self._pipeline['pipelineConfig']['stopEventStages'] = [stage_instance]
-        self._set_pipeline_configuration('stopEventStage', self._stage_to_configuration_name(stage_instance))
 
+        # We need instanceName in the form <instance_name>_StartEventStage and not <instance_name>_01
+        instance_name_split_list = stage_instance['instanceName'].split('_')
+        instance_name_split_list[-1] = 'StopEventStage'
+        stage_instance['instanceName'] = '_'.join(instance_name_split_list)
+
+        self._pipeline[self._config_key]['stopEventStages'] = [stage_instance]
+        self._set_pipeline_configuration('stopEventStage', self._stage_to_configuration_name(stage_instance))
         return self._all_stages.get(stage_instance['stageName'], Stage)(
             stage=stage_instance,
             output_streams=stage_definition.get('outputStreams', 0),

{streamsets-6.6.1rc1 → streamsets-6.6.2.dev0}/streamsets/sdk/utils.py

@@ -1058,3 +1058,35 @@ def validate_pipeline_stages(pipeline):
     for stage in pipeline.stages:
         if stage._data['library'] not in engine_libraries:
             raise ValueError("Stage {} does not exist within Engine Libraries".format(stage.instance_name))
+
+
+def get_accepted_labels_libraries_and_names(pipeline_builder, field_name):
+    # We need the  field_name library definition in order to get accepted libraries, names & labels.
+    # The accepted labels and associated java_name values are of the form:
+    # {'labels': [label1 (library: Library_name1), label2 (Library: library_name2)],
+    #  'values': [java_name_for_label1, java_name_for_label2]}
+    config_definition = next(
+        (
+            config.get('model')
+            for config in pipeline_builder._definitions['pipeline'][0]['configDefinitions']
+            if config['fieldName'] == field_name
+        ),
+        {},
+    )
+    if not config_definition:
+        raise RuntimeError('Could not pull the {} library definitions'.format(field_name))
+
+    # Create accepted lists
+    accepted_labels = []
+    accepted_libraries = []
+    accepted_java_names = []
+    for idx, field in enumerate(config_definition['labels']):
+        label_name = field.split(' (')[0]
+        accepted_labels.append(label_name)
+
+        library_name = re.search(r'Library:\s*([^)]+)\)', field)
+        accepted_libraries.append(library_name.group(1).strip())
+
+        accepted_java_names.append(config_definition['values'][idx])
+
+    return accepted_labels, accepted_libraries, accepted_java_names

{streamsets-6.6.1rc1 → streamsets-6.6.2.dev0/streamsets.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: streamsets
-Version: 6.6.1rc1
+Version: 6.6.2.dev0
 Summary: A Python SDK for StreamSets
 Author: StreamSets Inc
 Classifier: Development Status :: 5 - Production/Stable

{streamsets-6.6.1rc1 → streamsets-6.6.2.dev0}/tests/integration/test_pipelines.py

@@ -2,12 +2,10 @@
 
 # fmt: off
 import json
-import logging
 import os
 import re
 import time
 from datetime import datetime
-from io import StringIO
 
 import pytest
 
@@ -1060,26 +1058,35 @@ def test_call_get_pipeline_builder_with_disparate_engine_type_and_engine(sch, sc
         sch.get_pipeline_builder(engine_type, sch_authoring_id)
 
 
-def test_pipeline_pagination(sch, sch_authoring_sdc_id):
-    pipeline_builder1 = sch.get_pipeline_builder(engine_type='data_collector', engine_id=sch_authoring_sdc_id)
-    published_pipeline1 = pipeline_builder1.build(title='TEST_1')
-    sch.publish_pipeline(published_pipeline1)
+def test_event_and_origin_for_pipeline(sch, sample_pipeline):
 
-    pipeline_builder2 = sch.get_pipeline_builder(engine_type='data_collector', engine_id=sch_authoring_sdc_id)
-    published_pipeline2 = pipeline_builder2.build(title='TEST_1')
-    sch.publish_pipeline(published_pipeline2)
+    # Test start events
+    start_event = sample_pipeline.set_start_event(label='JDBC Query')
+    assert start_event._data['uiInfo']['label'] == 'JDBC Query 1'
+    start_event.configuration.jdbc_connection_string = 'foo'
+    assert start_event.configuration.jdbc_connection_string == 'foo'
 
-    pipeline_builder3 = sch.get_pipeline_builder(engine_type='data_collector', engine_id=sch_authoring_sdc_id)
-    published_pipeline3 = pipeline_builder3.build(title='TEST_2')
-    sch.publish_pipeline(published_pipeline3)
+    # Test stop events
+    stop_event = sample_pipeline.set_stop_event(label='JDBC Query')
+    assert stop_event._data['uiInfo']['label'] == 'JDBC Query 1'
+    stop_event.configuration.jdbc_connection_string = 'foo'
+    assert stop_event.configuration.jdbc_connection_string == 'foo'
 
-    log_stream = StringIO()
-    handler = logging.StreamHandler(log_stream)
-    handler.setLevel(logging.DEBUG)
-    logging.getLogger().addHandler(handler)
+    # Test stop events
+    test_origin = sample_pipeline.set_test_origin(label='File Tail')
+    assert test_origin._data['uiInfo']['label'] == 'File Tail 1'
+    test_origin.configuration.on_record_error = 'foo'
+    assert test_origin.configuration.on_record_error == 'foo'
 
-    sch.pipelines.get_all(search="name==*TEST_1*", len=1)
+    # Test if changes reflected on Platform
+    sch.publish_pipeline(pipeline=sample_pipeline)
+    sample_pipeline = sch.pipelines.get(pipeline_id=sample_pipeline.pipeline_id)
 
-    handler.flush()
-    log_contents = log_stream.getvalue()
-    assert log_contents.find("Current offset was negative") == -1
+    assert sample_pipeline.start_event._data['uiInfo']['label'] == 'JDBC Query 1'
+    assert sample_pipeline.start_event.configuration.jdbc_connection_string == 'foo'
+
+    assert sample_pipeline.stop_event._data['uiInfo']['label'] == 'JDBC Query 1'
+    assert sample_pipeline.stop_event.configuration.jdbc_connection_string == 'foo'
+
+    assert sample_pipeline.test_origin._data['uiInfo']['label'] == 'File Tail 1'
+    assert sample_pipeline.test_origin.configuration.on_record_error == 'foo'