streamsets 6.3__tar.gz → 6.4.0__tar.gz

{streamsets-6.3 → streamsets-6.4.0}/HISTORY.rst

@@ -3,6 +3,23 @@
 
 History
 =======
+6.4.0 (July 2024)
+-----------------
+* Added :py:meth:`streamsets.sdk.ControlHub.clone_deployment` method to easily a clone a deployment.
+
+* Added :py:meth:`streamsets.sdk.ControlHub.get_kubernetes_environment_yaml` method to fetch a Kubernetes environment's YAML.
+
+* Added :py:meth:`streamsets.sdk.sch_models.Pipeline.add_fragment` method which lets users add fragments to existing pipelines.
+
+* Added :py:meth:`streamsets.sdk.sch_models.Pipeline.get_jobs_using_pipeline` method which returns all the jobs that use the pipeline.
+
+* Added :py:meth:`streamsets.sdk.sch_models.JobSequence.delete_history_logs` method which can delete history logs for the job sequence.
+
+* Resolved a bug that did not let users update permissions for deployments correctly.
+
+* Bug fixes and improvements.
+
+
 6.3.0 (May 2024)
 ----------------
 * The :py:meth:`streamsets.sdk.ControlHub.get_self_managed_deployment_install_script` method now accepts a parameter ``java_version`` that can be used to specify which java version to be used when generating an install script.

{streamsets-6.3 → streamsets-6.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: streamsets
-Version: 6.3
+Version: 6.4.0
 Summary: A Python SDK for StreamSets
 Author: StreamSets Inc
 Classifier: Development Status :: 5 - Production/Stable

streamsets-6.4.0/docs/_static/full-ibm-streamsets-logo.svg

@@ -0,0 +1,3 @@
+<svg width="181" height="17" viewBox="0 0 181 17" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M0.5 16.7209V15.0733H2.86249V1.93743H0.5V0.289742H7.24863V1.93743H4.88614V15.0733H7.24863V16.7209H0.5ZM10.9361 0.289742H17.8529C20.6244 0.289742 22.3372 1.96025 22.3372 4.52762C22.3372 6.97633 20.7225 7.84581 19.6125 8.01241V8.1539C20.8183 8.22464 22.9144 9.23562 22.9144 11.9673C22.9144 14.6739 21.059 16.7232 18.575 16.7232H10.9337V0.289742H10.9361ZM12.9621 7.39852H17.6146C19.2059 7.39852 20.1687 6.57468 20.1687 5.11412V4.31311C20.1687 2.85255 19.2036 2.02871 17.6146 2.02871H12.9621V7.39852ZM12.9621 14.9797H17.9511C19.6873 14.9797 20.7482 14.0851 20.7482 12.4374V11.6364C20.7482 9.98871 19.6873 9.09413 17.9511 9.09413H12.9621V14.9797ZM26.6766 0.289742H29.3756L34.1964 9.13977H34.3179L39.1621 0.289742H41.7652V16.7209H39.7883V2.87994H39.6668L38.2437 5.63445L34.2174 12.7911L30.1912 5.63445L28.7681 2.87994H28.6465V16.7209H26.6696V0.289742H26.6766ZM49.6472 14.343L52.0588 11.965C53.1921 13.2133 54.6386 13.8478 56.2767 13.8478C58.0596 13.8478 58.9289 13.0718 58.9289 11.8692C58.9289 10.9746 58.5433 10.3151 56.9754 10.1028L55.3116 9.89059C51.769 9.44329 50.1543 7.74768 50.1543 4.99317C50.1543 2.05153 52.5168 0.00219727 56.4683 0.00219727C58.999 0.00219727 60.9269 0.82604 62.3009 2.30941L59.866 4.71019C59.0714 3.79278 57.9848 3.15607 56.251 3.15607C54.6129 3.15607 53.816 3.76768 53.816 4.75584C53.816 5.88548 54.4189 6.30995 55.8163 6.52219L57.4801 6.78007C60.9269 7.32093 62.5906 8.82712 62.5906 11.6524C62.5906 14.8062 60.2282 16.9971 56.0336 16.9971C53.094 17.0039 51.0937 15.8971 49.6472 14.343ZM69.7505 16.7209C67.2923 16.7209 65.9907 15.4726 65.9907 13.1198V7.0699H64.2077V4.36331H65.1004C66.0654 4.36331 66.3529 3.8932 66.3529 3.04425V1.02002H69.5589V4.36331H72.0663V7.0699H69.5589V14.0144H71.8723V16.7209H69.7505ZM74.5456 16.7209V4.36331H78.1139V7.04707H78.2354C78.5252 5.68237 79.4879 4.36331 81.4648 4.36331H82.0911V7.61076H81.1984C79.1257 7.61076 78.1139 8.05805 78.1139 9.4935V16.7209H74.5456ZM83.3693 10.5296C83.3693 6.6226 85.4911 4.08033 89.3001 4.08033C93.4946 4.08033 95.1817 7.04707 95.1817 10.3881V11.4242H87.0591V11.6136C87.0591 13.2384 87.9517 14.2745 89.8562 14.2745C91.3494 14.2745 92.1696 13.5922 92.9665 12.7911L94.7494 14.9569C93.6161 16.228 91.7607 17.0039 89.4473 17.0039C85.5612 17.0039 83.3693 14.4845 83.3693 10.5296ZM87.0567 9.18769V9.37711H91.492V9.16487C91.492 7.65868 90.7933 6.64542 89.3702 6.64542C87.9471 6.64542 87.0567 7.65868 87.0567 9.18769ZM107.062 16.7209C105.784 16.7209 104.868 15.8743 104.65 14.5552H104.506C104.12 16.155 102.723 17.0039 100.795 17.0039C98.264 17.0039 96.8666 15.5434 96.8666 13.4028C96.8666 10.7897 98.9393 9.54142 102.29 9.54142H104.291V8.7655C104.291 7.56511 103.688 6.81202 102.194 6.81202C100.774 6.81202 100.026 7.51719 99.4721 8.27257L97.3503 6.43547C98.3621 4.97491 99.8086 4.08261 102.461 4.08261C106.029 4.08261 107.861 5.68237 107.861 8.60119V14.0144H109.041V16.7209H107.062ZM104.291 12.9783V11.5177H102.554C101.18 11.5177 100.458 12.0129 100.458 12.9075V13.2613C100.458 14.1558 101.061 14.5803 102.122 14.5803C103.302 14.578 104.291 14.1079 104.291 12.9783ZM111.425 16.7209V4.36331H114.993V6.45829H115.138C115.596 5.13923 116.657 4.08033 118.536 4.08033C120.272 4.08033 121.524 4.95209 122.078 6.57467H122.151C122.609 5.18487 124.078 4.08033 126.006 4.08033C128.441 4.08033 129.79 5.86951 129.79 8.97774V16.7209H126.221V9.25844C126.221 7.65868 125.667 6.90558 124.485 6.90558C123.424 6.90558 122.389 7.49437 122.389 8.71758V16.7209H118.821V9.25844C118.821 7.65868 118.241 6.90558 117.084 6.90558C116.047 6.90558 114.988 7.49437 114.988 8.71758V16.7209H111.425ZM132.227 14.343L134.636 11.965C135.769 13.2133 137.216 13.8478 138.854 13.8478C140.637 13.8478 141.506 13.0718 141.506 11.8692C141.506 10.9746 141.121 10.3151 139.555 10.1028L137.891 9.89059C134.349 9.44329 132.734 7.74768 132.734 4.99317C132.734 2.05153 135.096 0.00219727 139.05 0.00219727C141.581 0.00219727 143.509 0.82604 144.883 2.30941L142.448 4.71019C141.653 3.79278 140.567 3.15607 138.833 3.15607C137.195 3.15607 136.398 3.76768 136.398 4.75584C136.398 5.88548 137.001 6.30995 138.398 6.52219L140.062 6.78007C143.509 7.32093 145.173 8.82712 145.173 11.6524C145.173 14.8062 142.81 16.9971 138.616 16.9971C135.674 17.0039 133.673 15.8971 132.227 14.343ZM147.22 10.5296C147.22 6.6226 149.341 4.08033 153.148 4.08033C157.342 4.08033 159.03 7.04707 159.03 10.3881V11.4242H150.907V11.6136C150.907 13.2384 151.8 14.2745 153.704 14.2745C155.197 14.2745 156.018 13.5922 156.814 12.7911L158.597 14.9569C157.464 16.228 155.609 17.0039 153.295 17.0039C149.411 17.0039 147.22 14.4845 147.22 10.5296ZM150.907 9.18769V9.37711H155.342V9.16487C155.342 7.65868 154.644 6.64542 153.22 6.64542C151.797 6.64542 150.907 7.65868 150.907 9.18769ZM165.996 16.7209C163.537 16.7209 162.236 15.4726 162.236 13.1198V7.0699H160.453V4.36331H161.345C162.308 4.36331 162.598 3.8932 162.598 3.04425V1.02002H165.804V4.36331H168.311V7.0699H165.804V14.0144H168.117V16.7209H165.996ZM169.755 14.7903L171.852 12.7181C172.744 13.6835 173.828 14.2951 175.203 14.2951C176.6 14.2951 177.081 13.8478 177.081 13.1175C177.081 12.5059 176.696 12.2001 175.803 12.0814L174.429 11.9171C171.559 11.5634 170.162 10.3402 170.162 8.10369C170.162 5.72573 172.067 4.07805 175.079 4.07805C177.731 4.07805 178.96 4.78322 180.236 6.00871L178.187 8.05577C177.392 7.27985 176.331 6.78463 175.27 6.78463C174.065 6.78463 173.583 7.2091 173.583 7.82071C173.583 8.48024 173.873 8.83397 174.908 8.97318L176.331 9.16259C179.079 9.51632 180.5 10.7395 180.5 12.8824C180.5 15.2604 178.331 17.0016 175.076 17.0016C172.648 17.0039 171.057 16.2029 169.755 14.7903Z" fill="#222222"/>
+</svg>

{streamsets-6.3 → streamsets-6.4.0}/docs/build/creating_pipelines.rst

@@ -74,13 +74,16 @@ you can do this with ``Dev Raw Data Source`` origin and ``Trash`` destination, r
   ``Dev Raw Data Source`` origin cannot be used in Transformer for Snowflake pipelines.
   Instead, use ``Snowflake Table`` or ``Snowflake Query``
 
-Sometimes, it is useful to pass in the ``type`` of stage we want to use. If we look at the image below, we can see
-that there are two types of ``Azure Data Lake Storage Gen2``. One of type ``Processor`` and another of type ``Origin``.
+Sometimes it is necessary to pass in the ``type`` of stage you want to use. If you look at the image below for example, you'll see that there are two types of ``Azure Data Lake Storage Gen2``.
+One of type ``Destination`` and another of type ``Origin``.
 
 .. image:: ../_static/images/build/stages_with_different_types.png
 |
 
-If we want to specify the ``type`` of stage we want to add through the sdk we can pass the optional ``type`` parameter
+.. note::
+  There are four possible value for ``type`` namely, ``'origin'``, ``'processor'``, ``'executor'`` and ``'destination'``.
+
+If you want to specify the ``type`` of stage to add through via sdk, you can pass the optional ``type`` parameter
 to the :py:meth:`streamsets.sdk.sch_models.PipelineBuilder.add_stage` method.
 
 .. code-block:: python
@@ -88,8 +91,10 @@ to the :py:meth:`streamsets.sdk.sch_models.PipelineBuilder.add_stage` method.
     azure_data_lake_origin = pipeline_builder.add_stage('Azure Data Lake Storage Gen2', type='origin')
     azure_data_lake_processor = pipeline_builder.add_stage('Azure Data Lake Storage Gen2', type='processor')
 
-.. note::
-  There are four possible value for ``type`` namely, ``'origin'``, ``'processor'``, ``'executor'`` and ``'destination'``.
+.. warning::
+  When a stage is created by passing a stage label to :py:meth:`streamsets.sdk.sch_models.PipelineBuilder.add_stage`, the method picks the first stage that matches that label.
+  The same label can be used by multiple stages (e.g., the Amazon S3 origin and the Amazon S3 destination both use the `'Amazon S3'` label).
+  As a best practice, you should always specify the ``type`` parameter for a stage to guarantee the correct stage is selected.
 
 Connecting the Stages
 ~~~~~~~~~~~~~~~~~~~~~

{streamsets-6.3 → streamsets-6.4.0}/docs/conf.py

@@ -112,9 +112,9 @@ pygments_style = 'sphinx'
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 html_theme = 'sphinx_book_theme'
-html_logo = '_static/full-streamsets-logo.svg'
+html_logo = '_static/full-ibm-streamsets-logo.svg'
 
-html_title = 'StreamSets Platform SDK for Python Version {}'.format(version)
+html_title = 'Platform SDK for Python Version {}'.format(version)
 
 show_navbar_depth = 2
 

{streamsets-6.3 → streamsets-6.4.0}/docs/manage/users_and_groups/permissions.rst

@@ -188,6 +188,9 @@ the actions as needed:
     # pipeline.acl.permissions.get(subject_id=group.id).actions
     ['READ']
 
+.. note::
+  ``ACL Permissions`` are limited to following actions:
+  ``["READ"], ["READ", "WRITE"], ["READ", "EXECUTE"], or ["READ", "WRITE", "EXECUTE"]``.
 Removing ACL permissions on an object
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

{streamsets-6.3 → streamsets-6.4.0}/evergreen/.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: check-case-conflict # checks for files that would conflict in case-insensitive filesystems
       - id: trailing-whitespace # trims trailing whitespace.
   - repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 24.4.1
     hooks:
       - id: black # Black: The uncompromising Python code formatter
         args: ['--config', 'python/evergreen/pyproject.toml' ]

{streamsets-6.3 → streamsets-6.4.0}/evergreen/pyproject.toml

@@ -1,7 +1,6 @@
 # Format and Linter configuration
 
 [tool.black]
-single-quote = true
 line-length = 120
 skip-string-normalization = true
 exclude = '''

{streamsets-6.3 → streamsets-6.4.0}/requirements.txt

@@ -7,5 +7,4 @@ Cython==0.29.30
 inflection==0.3.1
 javaproperties==0.4.0
 PyYAML==5.4.1
-pycryptodome==3.9.0
 requests==2.27

{streamsets-6.3 → streamsets-6.4.0}/streamsets/sdk/__version__.py

@@ -1,3 +1,3 @@
 # Copyright 2024 StreamSets Inc.
 
-__version__ = "6.3"
+__version__ = "6.4.0"

{streamsets-6.3 → streamsets-6.4.0}/streamsets/sdk/aster_api.py

@@ -7,9 +7,10 @@ import logging
 
 import requests
 import urllib3
+from requests.adapters import HTTPAdapter
 
 from .exceptions import BadRequestError, InternalServerError, UnprocessableEntityError
-from .utils import create_aster_envelope, get_params, join_url_parts
+from .utils import create_aster_envelope, get_params, join_url_parts, retry_on_connection_error
 
 # fmt: off
 
@@ -62,13 +63,18 @@ class ApiClient(object):
         self._sch_auth_token = sch_auth_token
         self._sch_credential_id = sch_credential_id
         self._api_version = api_version
+
         self._session = requests.Session()
+        self._session.mount('http://', HTTPAdapter(pool_connections=1))
+        self._session.mount('https://', HTTPAdapter(pool_connections=1))
         self._session.headers.update(REQUIRED_HEADERS)
         if headers:
             self._session.headers.update(headers)
+
         if session_attributes:
             for attribute, value in session_attributes.items():
                 setattr(self._session, attribute, value)
+
         if not self._session.verify:
             # If we disable SSL cert verification, we should disable warnings about having disabled it.
             urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@@ -1100,33 +1106,51 @@ class ApiClient(object):
             org (:obj:`str`, optional): Organization id. Default: ``None``
 
         Returns:
-            An instance of :py:class:`streamsets.sdk.sch_api.Command`.
+            An instance of :py:class:`streamsets.sdk.aster_api.Command`.
         """
         params = get_params(parameters=locals(), exclusions=('self',))
         endpoint = '/v{}/saml/login-url'.format(self._api_version)
         response = self._get(app='security', endpoint=endpoint, params=params)
         return Command(self, response)
 
+    def trigger_saml_automated_task(self):
+        """Trigger the automated task that generates banners for expiring SAML certificates.
+           Call can only be made by Sys-Admins.
+
+        Returns:
+            An instance of :py:class:`streamsets.sdk.aster_api.Command`.
+        """
+        response = self._post(
+            app='security',
+            endpoint='/v{}/admin/task/saml-cert.trigger'.format(self._api_version)
+        )
+        return Command(self, response)
+
+    @retry_on_connection_error
     def _delete(self, app, endpoint, params=None):
         url = join_url_parts(self._base_url, '/api/{}'.format(app), endpoint)
         response = self._session.delete(url, params=params or {})
         self._handle_http_error(response)
         return response
 
+    @retry_on_connection_error
     def _get(self, app, endpoint, params=None):
         url = join_url_parts(self._base_url, '/api/{}'.format(app), endpoint)
         response = self._session.get(url, params=params or {})
         self._handle_http_error(response)
         return response
 
+    @retry_on_connection_error
     def _post(self, app, endpoint, params=None, data=None, files=None, headers=None):
         url = join_url_parts(self._base_url, '/api/{}'.format(app), endpoint)
         if data and not isinstance(data, str):
             data = json.dumps(data or {})
+
         response = self._session.post(url, params=params or {}, data=data, files=files, headers=headers)
         self._handle_http_error(response)
         return response
 
+    @retry_on_connection_error
     def _put(self, app, endpoint, params=None, data=None):
         url = join_url_parts(self._base_url, '/api/{}'.format(app), endpoint)
         body = {'data': (data or {})}

{streamsets-6.3 → streamsets-6.4.0}/streamsets/sdk/constants.py

@@ -28,6 +28,8 @@ EXECUTOR_TO_DEPLOYMENT_MAP = {
     TRANSFORMER_EXECUTOR_TYPE: TRANSFORMER_DEPLOYMENT_TYPE,
 }
 
+ENGINELESS_ENGINE_ID = 'SYSTEM_DESIGNER'
+
 StageConfigurationProperty = collections.namedtuple('StageConfigurationProperty', ['config_name'])
 ServiceConfigurationProperty = collections.namedtuple('ServiceConfigurationProperty', ['service_name', 'config_name'])
 

{streamsets-6.3 → streamsets-6.4.0}/streamsets/sdk/exceptions.py

@@ -189,3 +189,10 @@ class ServiceDefinitionNotFound(Exception):
     def __init__(self, message):
         super().__init__('{}'.format(message))
         self.message = message
+
+
+class EnginelessError(Exception):
+    """Publishing an engineless designed pipeline error"""
+
+    def __init__(self, message):
+        self.message = message