streamsets 6.5.0rc1__tar.gz → 6.6.0.dev0__tar.gz

{streamsets-6.5.0rc1 → streamsets-6.6.0.dev0}/HISTORY.rst

@@ -3,6 +3,24 @@
 
 History
 =======
+6.5.0 (November 2024)
+-----------------
+* The :py:class:`streamsets.sdk.sch_models.Step` class now supports Finish Conditions and has a new attribute ``name`` which will allow retrieving and setting the name of a step.
+
+* Added support for running an arbitrary step using the The :py:meth:`streamsets.sdk.ControlHub.run_job_sequence` method.
+
+* Extended AQL functionality to now support searching on :py:class:`streamsets.sdk.sch_models.JobSequence` objects.
+
+* Updated API calls to be more efficient by changing the default page size to `250` instead of `50`.
+
+* Added SDK support for creating connections without an authoring engine specified (engineless connections).
+
+* Added SDK support for stopping a Deployment through the :py:meth:`streamsets.sdk.ControlHub.delete_deployment` method.
+
+* Added the ability to specify the ``install_type`` for a self-managed deployment when retrieving the install script.
+
+* Bug fixes and improvements.
+
 6.4.0 (July 2024)
 -----------------
 * Added :py:meth:`streamsets.sdk.ControlHub.clone_deployment` method to easily a clone a deployment.
@@ -19,7 +37,6 @@ History
 
 * Bug fixes and improvements.
 
-
 6.3.0 (May 2024)
 ----------------
 * The :py:meth:`streamsets.sdk.ControlHub.get_self_managed_deployment_install_script` method now accepts a parameter ``java_version`` that can be used to specify which java version to be used when generating an install script.

{streamsets-6.5.0rc1 → streamsets-6.6.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: streamsets
-Version: 6.5.0rc1
+Version: 6.6.0.dev0
 Summary: A Python SDK for StreamSets
 Author: StreamSets Inc
 Classifier: Development Status :: 5 - Production/Stable

{streamsets-6.5.0rc1 → streamsets-6.6.0.dev0}/docs/usage/set_up/connections/connections.rst

@@ -40,7 +40,7 @@ To create a new Connection and add it to the Platform via the SDK, you will need
 To instantiate a builder instance, use the :py:meth:`streamsets.sdk.ControlHub.get_connection_builder` method.
 
 Once you've instantiated the builder, you will call the :py:meth:`streamsets.sdk.sch_models.ConnectionBuilder.build` method, along with a few parameters, to create the desired :py:class:`streamsets.sdk.sch_models.Connection` instance.
-The parameters you'll need to provide for the Connection are ``title``, ``connection_type``, ``authoring_data_collector``, and (optionally) ``tags`` to associate with the Connection:
+The parameters you'll need to provide for the Connection are ``title``, ``connection_type``, ``authoring_data_collector`` (optional), and ``tags`` (optional) to associate with the Connection:
 
 .. code-block:: python
 
@@ -54,6 +54,9 @@ The parameters you'll need to provide for the Connection are ``title``, ``connec
                                           authoring_data_collector=engine,
                                           tags=['sdk_example', 's3_connection'])
 
+.. tip::
+  Supplying an existing, responsive :py:class:`streamsets.sdk.sch_models.Engine` instance via the ``authoring_data_collector`` parameter is the recommended best practice when building a connection.
+
 .. tip::
   The available options for the ``connection_type`` in the :py:meth:`streamsets.sdk.sch_models.ConnectionBuilder.build` method are:
   ``'STREAMSETS_AWS_EMR_CLUSTER'``, ``'STREAMSETS_MYSQL'``, ``'STREAMSETS_SNOWFLAKE'``, ``'STREAMSETS_COAP_CLIENT'``, ``'STREAMSETS_OPC_UA_CLIENT'``, ``'STREAMSETS_GOOGLE_PUB_SUB'``, ``'STREAMSETS_MQTT'``, ``'STREAMSETS_POSTGRES'``, ``'STREAMSETS_GOOGLE_CLOUD_STORAGE'``, ``'STREAMSETS_AWS_REDSHIFT'``, ``'STREAMSETS_GOOGLE_BIG_QUERY'``, ``'STREAMSETS_ORACLE'``, ``'STREAMSETS_AWS_S3'``, ``'STREAMSETS_REMOTE_FILE'``, ``'STREAMSETS_SQLSERVER'``, ``'STREAMSETS_AWS_SQS'``, ``'STREAMSETS_SNOWPIPE'``, and ``'STREAMSETS_JDBC'``.

streamsets-6.6.0.dev0/streamsets/sdk/__version__.py

@@ -0,0 +1,5 @@
+#  IBM Confidential
+#  PID 5900-BAF
+#  Copyright StreamSets Inc., an IBM Company 2024
+
+__version__ = "6.6.0-dev"

{streamsets-6.5.0rc1 → streamsets-6.6.0.dev0}/streamsets/sdk/sch.py

@@ -40,11 +40,12 @@ from .sch_models import (
     Environments, GCEDeployment, GCPEnvironment, GroupBuilder, Groups, Job, JobBuilder, Jobs, JobSequence,
     JobSequenceBuilder, JobSequences, KubernetesDeployment, KubernetesEnvironment, LegacyDeploymentBuilder,
     LegacyDeployments, LoginAudits, MeteringUsage, NotImplementedDeployment, NotImplementedEnvironment,
-    OrganizationBuilder, Organizations, Pipeline, PipelineBuilder, PipelineLabels, Pipelines, ProtectionMethodBuilder,
-    ProtectionPolicies, ProtectionPolicy, ProtectionPolicyBuilder, ProvisioningAgents, ReportDefinitions, SAQLSearch,
-    SAQLSearchBuilder, SAQLSearches, ScheduledTask, ScheduledTaskActions, ScheduledTaskBuilder, ScheduledTasks,
-    SelfManagedDeployment, SelfManagedEnvironment, StPipelineBuilder, SubscriptionAudit, SubscriptionBuilder,
-    Subscriptions, Topologies, Topology, TopologyBuilder, UserBuilder, Users,
+    OrganizationBuilder, Organizations, Pipeline, PipelineBuilder, PipelineLabels, Pipelines, Project, ProjectBuilder,
+    Projects, ProtectionMethodBuilder, ProtectionPolicies, ProtectionPolicy, ProtectionPolicyBuilder,
+    ProvisioningAgents, SAQLSearch, SAQLSearchBuilder, SAQLSearches, ScheduledTask, ScheduledTaskActions,
+    ScheduledTaskBuilder, ScheduledTasks, SelfManagedDeployment, SelfManagedEnvironment,
+    StPipelineBuilder, SubscriptionAudit, SubscriptionBuilder, Subscriptions, Topologies, Topology, TopologyBuilder,
+    UserBuilder, Users,
 )
 from .utils import (
     SDC_DEFAULT_EXECUTION_MODE, TRANSFORMER_EXECUTION_MODES, SeekableList, get_stage_library_display_name_from_library,
@@ -181,6 +182,9 @@ class ControlHub:
 
         self._en_translations = self.api_client.get_translations_json()
 
+        # store which project we are running in, if any
+        self._current_project = None
+
         self._data_collectors = {}
         self._transformers = {}
         thread = threading.Thread(target=self._call_data_collectors)
@@ -403,6 +407,124 @@ class ControlHub:
             engine_start_up_time=authoring_engine._data["startUpTime"] if authoring_engine else 0,
         )
 
+    def get_snowflake_pipeline_user_defined_functions(self, pipeline, filter_text=None, tunneling=False):
+        """Return user defined functions for the pipeline
+
+        Args:
+            pipeline (:py:class:`streamsets.sdk.sch_models.Pipeline`): Pipeline object.
+            filter_text (:obj:`str`: optional): Filter response text. Default: ``None``.
+            tunneling (:obj:`bool`: optional): Enable tunneling for hosted instances. Default: ``False``.
+
+        Rreturns:
+            An instance of :py:class:`streamsets.sdk.sch_api.Command`.
+        """
+        engine_id = None
+        tunneling_instance_id = None
+        if tunneling:
+            engine_id = pipeline.engine_id
+            tunneling_instance_id_request = self.api_client.get_tunneling_instance_id(engine_id)
+            tunneling_instance_id = tunneling_instance_id_request.response.json().get('instanceId')
+
+        pipeline_fqn = f'{pipeline.pipeline_id.split(":")[0]}__{pipeline.organization}__{pipeline.committer}'
+        response = self.api_client.post_snowflake_pipeline_user_defined_functions_search_start(
+            pipeline_id=pipeline_fqn,
+            engine_id=engine_id,
+            rev=pipeline.commit_id,
+            filter_text=filter_text,
+            tunneling_instance_id=tunneling_instance_id,
+        )
+        id = response.response.json().get('id', '')
+        responses = []
+
+        def failure(timeout=None, exception=False):
+            self.api_client.delete_snowflake_pipeline_user_defined_fucntions_search_cancel(
+                pipeline_id=pipeline_fqn,
+                engine_id=engine_id,
+                id=id,
+                rev=pipeline.commit_id,
+                tunneling_instance_id=tunneling_instance_id,
+            )
+            if exception:
+                raise ValueError('An ERROR status was reached before retrieving UDFs.')
+            else:
+                raise TimeoutError(f'Timeout reached after {timeout} seconds while searching for UDFs.')
+
+        def condition(responses=[]):
+            response = self.api_client.get_snowflake_pipeline_user_defined_functions_search_status(
+                pipeline_id=pipeline_fqn,
+                id=id,
+                engine_id=engine_id,
+                rev=pipeline.commit_id,
+                tunneling_instance_id=tunneling_instance_id,
+            )
+            status = response.response.json().get('status', '')
+            responses.append(response)
+            if status == 'ERROR':
+                failure(exception=True)
+            return status == 'FINISHED'
+
+        wait_for_condition(condition, condition_kwargs={'responses': responses}, failure=failure, timeout=600)
+        return responses[-1]
+
+    def get_snowflake_pipeline_procedures(self, pipeline, filter_text=None, tunneling=False):
+        """Return procedures for the pipeline
+
+        Args:
+            pipeline (:py:class:`streamsets.sdk.sch_models.Pipeline`): Pipeline object.
+            filter_text (:obj:`str`: optional): Filter response text. Default: ``None``.
+            tunneling (:obj:`bool`: optional): Enable tunneling for hosted instances. Default: ``False``.
+
+        Rreturns:
+            An instance of :py:class:`streamsets.sdk.sch_api.Command`.
+        """
+        engine_id = None
+        tunneling_instance_id = None
+        if tunneling:
+            engine_id = pipeline.engine_id
+            tunneling_instance_id_request = self.api_client.get_tunneling_instance_id(engine_id)
+            tunneling_instance_id = tunneling_instance_id_request.response.json().get('instanceId')
+
+        pipeline_fqn = f'{pipeline.pipeline_id.split(":")[0]}__{pipeline.organization}__{pipeline.committer}'
+        response = self.api_client.post_snowflake_pipeline_procedures_search_start(
+            pipeline_id=pipeline_fqn,
+            engine_id=engine_id,
+            rev=pipeline.commit_id,
+            fitler_text=filter_text,
+            tunneling_instance_id=tunneling_instance_id,
+        )
+        id = response.response.json().get('id', '')
+        responses = []
+
+        def failure(timeout=None, exception=False):
+            self.api_client.delete_snowflake_pipeline_procedures_search_cancel(
+                pipeline_id=pipeline_fqn,
+                id=id,
+                engine_id=engine_id,
+                rev=pipeline.commit_id,
+                tunneling_instance_id=tunneling_instance_id,
+            )
+            if exception:
+                raise ValueError('An ERROR status was reached before retrieving the pipeline procedures.')
+            else:
+                raise TimeoutError(f'Timeout reached after {timeout} seconds while searching for pipeline procedures.')
+
+        def condition(responses=[]):
+            response = self.api_client.get_snowflake_pipeline_procedures_search_status(
+                pipeline_id=pipeline_fqn,
+                engine_id=engine_id,
+                id=id,
+                rev=pipeline.commit_id,
+                tunneling_instance_id=tunneling_instance_id,
+            )
+            status = response.response.json().get('status', '')
+            if status == 'ERROR':
+                failure(exception=True)
+            responses.append(response)
+            return status == 'FINISHED'
+
+        wait_for_condition(condition, condition_kwargs={'responses': responses}, failure=failure, timeout=600)
+        return responses[-1]
+
     def publish_pipeline(self, pipeline, commit_message='New pipeline', draft=False, validate=False):
         """Publish a pipeline.
 
@@ -3423,54 +3545,6 @@ class ControlHub:
 
         return response
 
-    @property
-    def report_definitions(self):
-        """Report Definitions.
-
-        Returns:
-            An instance of :py:class:`streamsets.sdk.sch_models.ReportDefinitions`.
-        """
-        return ReportDefinitions(self)
-
-    def add_report_definition(self, report_definition):
-        """Add Report Definition to ControlHub.
-
-        Args:
-            report_definition (:py:class:`streamsets.sdk.sch_models.ReportDefinition`): Report Definition instance.
-
-        Returns:
-            An instance of :py:class:`streamsets.sdk.sch_api.Command`.
-        """
-        create_report_definition_command = self.api_client.create_new_report_definition(report_definition._data)
-        report_definition._data = create_report_definition_command.response.json()
-        return create_report_definition_command
-
-    def update_report_definition(self, report_definition):
-        """Update an existing Report Definition.
-
-        Args:
-            report_definition (:py:class:`streamsets.sdk.sch_models.ReportDefinition`): Report Definition instance.
-
-        Returns:
-            An instance of :py:class:`streamsets.sdk.sch_api.Command`.
-        """
-        update_report_definition_command = self.api_client.update_report_definition(
-            report_definition.id, report_definition._data
-        )
-        report_definition._data = update_report_definition_command.response.json()
-        return update_report_definition_command
-
-    def delete_report_definition(self, report_definition):
-        """Delete an existing Report Definition.
-
-        Args:
-            report_definition (:py:class:`streamsets.sdk.sch_models.ReportDefinition`): Report Definition instance.
-
-        Returns:
-            An instance of :py:class:`streamsets.sdk.sch_api.Command`.
-        """
-        return self.api_client.delete_report_definition(report_definition.id)
-
     def get_connection_builder(self):
         """Get a connection builder instance with which a connection can be created.
 
@@ -4750,3 +4824,103 @@ class ControlHub:
             'username': username,
         }
         return self.api_client.validate_snowflake_account(body=validate_snowflake_account_json)
+
+    @property
+    def projects(self):
+        """Projects.
+
+        Returns:
+            An instance of :py:class:`streamsets.sdk.sch_models.Projects`.
+        """
+        return Projects(self)
+
+    @property
+    def current_project(self):
+        """Which project we are currently running in.
+
+        Returns:
+            An instance of :py:class:`streamsets.sdk.sch_models.Project` or :obj:`None`.
+        """
+        return self._current_project
+
+    @current_project.setter
+    def current_project(self, val):
+        if val is not None:
+            if not isinstance(val, Project):
+                raise TypeError(
+                    "current_project should be of type :py:class:`streamsets.sdk.sch_models.Project` or `None`"
+                )
+            if val.id is None:
+                raise ValueError(
+                    "ID of the project passed is not set. " "Set it by adding it to platform via `add_project`"
+                )
+        self._current_project = val
+
+        # set the appropriate project ID in the API client. This will consumed while making calls to platform.
+        self.api_client.current_project_id = self.current_project.id if self.current_project else None
+
+    def add_project(self, project):
+        """Add a project to Control Hub.
+
+        Args:
+            project (:py:class:`streamsets.sdk.sch_models.Project`): An instance of the project to be added.
+
+        Returns:
+            An instance of :py:class:`streamsets.sdk.sch_api.Command`.
+        """
+        if not isinstance(project, Project):
+            raise TypeError("project should be of type Project.")
+
+        request_body = {'name': project.name, 'description': project.description}
+        command = self.api_client.create_project(body=request_body)
+
+        # overwrite the data into the same object.
+        project._data = command.response.json()
+
+        return command
+
+    def update_project(self, project):
+        """Update an existing project.
+
+        Args:
+            project (:py:class:`streamsets.sdk.sch_models.Project`): An instance of the project to be updated.
+
+        Returns:
+            An instance of :py:class:`streamsets.sdk.sch_api.Command`.
+        """
+        if not isinstance(project, Project):
+            raise TypeError("project should be of type Project.")
+
+        if project.id is None:
+            raise ValueError("project is not initialized. Add it to Control Hub using the `add_project` method.")
+
+        command = self.api_client.update_project(data=project._data, project_id=project.id)
+        return command
+
+    def delete_project(self, *projects):
+        """Delete one or more projects from Control Hub
+
+        Args: projects (:py:class:`streamsets.sdk.sch_models.Project`): One or more instances of
+                                                                        :py:class:`streamsets.sdk.sch_models.Project`.
+
+        Returns:
+            An instance of :py:class:`streamsets.sdk.sch_api.Command`.
+        """
+        if not all([isinstance(project, Project) for project in projects]):
+            raise TypeError("project should be of type Project.")
+
+        request_body = {'projectIds': [project.id for project in projects]}
+        command = self.api_client.delete_projects(body=request_body)
+
+        return command
+
+    def get_project_builder(self):
+        """Get a ProjectBuilder instance to create a project.
+
+        Returns:
+            An instance of :py:class:`streamsets.sdk.sch_models.ProjectBuilder`.
+        """
+        project_properties = self._security_api['definitions']['ProjectJson']['properties']
+        project = {property_: None for property_ in project_properties}
+
+        return ProjectBuilder(project=project, control_hub=self)