streamsets 6.2.0.dev0__tar.gz → 6.3__tar.gz

streamsets-6.3/.gitignore

@@ -0,0 +1,66 @@
+# Copyright 2017 StreamSets Inc.
+
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+*.c
+
+# Distribution / packaging
+.Python
+env/
+/venv/
+/build/
+develop-eggs/
+/dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+/*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+/.sphinx-build/
+/html/
+
+# PyBuilder
+target/
+
+# pyenv python configuration file
+.python-version*.pyc

streamsets-6.3/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.8-slim
+RUN apt-get update && \
+    apt-get -y install \
+    gcc \
+    git \
+    make
+WORKDIR /root
+ADD requirements.txt ./
+RUN pip3 install -r requirements.txt
+ADD . .
+RUN make setup
+RUN make build

{streamsets-6.2.0.dev0 → streamsets-6.3}/HISTORY.rst

@@ -3,6 +3,34 @@
 
 History
 =======
+6.3.0 (May 2024)
+----------------
+* The :py:meth:`streamsets.sdk.ControlHub.get_self_managed_deployment_install_script` method now accepts a parameter ``java_version`` that can be used to specify which java version to be used when generating an install script.
+
+* The :py:meth:`streamsets.sdk.ControlHub.add_scheduled_task` method raises a more detailed error when creating a :py:class:`streamsets.sdk.sch_models.ScheduledTask`object.
+
+* Support for Job Sequencing has been added. Refer to the :ref:`StreamSets SDK Job Sequencing Documentation <job_sequences>` for further details.
+
+* Bug fixes and improvements.
+
+6.2.0 (February 2024)
+---------------------
+* The :py:meth:`streamsets.sdk.ControlHub.verify_connection` method now accepts a parameter ``library`` that can be used to specify which library the connection should be verified against.
+
+* The :py:meth:`streamsets.sdk.ControlHub.publish_pipeline` method now accepts a parameter ``validate`` which can be used to validate a pipeline when saving it on Platform.
+
+* The :py:meth:`streamsets.sdk.sch_models.PipelineBuilder.build` method now accepts a parameter ``description`` which can be used to set the description of a pipeline.
+
+* The :py:meth:`streamsets.sdk.sch_models.DeploymentBuilder.build` method now accepts additional parameters that can be used to configure the deployment during creation.
+
+* The :py:class:`streamsets.sdk.sch_models.Job` class has a new attribute ``latest_committed_offsets`` which will retrieve latest committed offsets for the Job.
+
+* Fixed an error that did not render color icons in the Platform UI for pipelines generated in the SDK.
+
+* Fixed an error with :py:meth:`streamsets.sdk.ControlHub.activate_api_credential`, :py:meth:`streamsets.sdk.ControlHub.deactivate_api_credential` and :py:meth:`streamsets.sdk.ControlHub.rename_api_credential` that caused the authentication token to be regenerated.
+
+* Bug fixes and improvements.
+
 6.1.0 (November 2023)
 ---------------------
 * Allow direct replacement of a stage in a pipeline. This is achieved through the ``copy_inputs`` & ``copy_outputs`` methods under the :py:class:`streamsets.sdk.sch_models.SchSdcStage` and :py:class:`streamsets.sdk.sch_models.SchStStage` classes respectively.

streamsets-6.3/Makefile

@@ -0,0 +1,74 @@
+.PHONY: setup format lint test integration docs clean
+
+REQUIRED_ENV_VARS = SCH_TOKEN SCH_CREDENTIAL_ID SDC_VERSION TRANSFORMER_VERSION ASTER_URL
+
+all:
+	@echo "\033[1;32mTo run a specific target, use: make <target>\033[0m"
+	@echo "\033[1;32mAvailable targets:\033[0m"
+	@grep -E '^[a-zA-Z0-9_-]+:' Makefile | grep -Ev '^all:|^validate:' | sed -E 's/([^:]+):.*/- \1/g' | sort
+
+setup:
+	@pip3 install -r build_requirements.txt
+	@pip install -r test_requirements.txt
+
+validate:
+	@$(foreach env_var,$(REQUIRED_ENV_VARS), \
+		$(if $(value $(env_var)),, \
+			$(error Environment variable $(env_var) is not defined)))
+
+format:
+	black --config=evergreen/pyproject.toml streamsets/sdk tests
+	isort --settings-path=evergreen/pyproject.toml streamsets/sdk tests
+
+lint:
+	flake8 --config=evergreen/.flake8 streamsets/sdk
+
+test:
+ifeq ($(filter test,$(MAKECMDGOALS)),)
+	@pytest -v --disable-warnings --cov=streamsets/sdk tests/unit
+else
+	@pytest -v --disable-warnings --cov=streamsets/sdk -k "$(word 2,$(MAKECMDGOALS))" tests/unit
+endif
+
+
+integration: validate
+ifeq ($(filter integration,$(MAKECMDGOALS)),)
+	@pytest -v \
+        --cov=streamsets/sdk \
+        --disable-warnings \
+        --sch-credential-id=$$SCH_CREDENTIAL_ID \
+        --sch-token=$$SCH_TOKEN \
+        --sdc-version=$$SDC_VERSION \
+        --transformer-version=$$TRANSFORMER_VERSION \
+        --aster-url=$$ASTER_URL \
+        tests/integration
+else
+	@pytest -v \
+		--cov=streamsets/sdk \
+		--disable-warnings \
+		--sch-credential-id=$$SCH_CREDENTIAL_ID \
+		--sch-token=$$SCH_TOKEN \
+		--sdc-version=$$SDC_VERSION \
+		--transformer-version=$$TRANSFORMER_VERSION \
+		--aster-url=$$ASTER_URL \
+		-k "$(word 2,$(MAKECMDGOALS))" \
+		tests/integration
+endif
+
+docs:
+	@pip install -r docs/requirements.txt
+	sphinx-build -b html -d .sphinx-build docs html
+
+build:
+	@python3 -m build
+
+clean:
+	@rm -rf venv html build dist htmlcov .sphinx-build .coverage .eggs *.egg-info
+	@find . -type f -name '*.pyc' -delete
+	@find . -type f -name '*.pyo' -delete
+	@find . -type f -name '*_unit_*.xml' -delete
+	@find . -type f -name '*_integration_*.xml' -delete
+	@find . -type f -name 'coverage.xml' -delete
+	@find . -type d -name '__pycache__' -prune -exec rm -rf "{}" \;
+	@find . -type d -name '.pytest_cache' -prune -exec rm -rf "{}" \;
+	@find . -type f -name '*~' -delete

streamsets-6.3/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.1
+Name: streamsets
+Version: 6.3
+Summary: A Python SDK for StreamSets
+Author: StreamSets Inc
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.4
+Description-Content-Type: text/x-rst
+Requires-Dist: dpath==1.5.0
+Requires-Dist: inflection
+Requires-Dist: PyYAML
+Requires-Dist: requests

streamsets-6.3/build_requirements.txt

@@ -0,0 +1,2 @@
+setuptools==68.1.2
+build==1.0.3

streamsets-6.3/docs/_static/css/docs_theme.css

@@ -0,0 +1,67 @@
+/* StreamSets Python SDK CSS file */
+
+/* FONTS
+------------------------------------------------------- */
+@import url(https://fonts.googleapis.com/css?family=Lato:300,400,700,900);
+@import url(https://fonts.googleapis.com/css?family=Montserrat:400,700);
+
+
+/* GENERAL
+------------------------------------------------------- */
+body {
+	font-family: Verdana, Geneva, sans-serif;
+	font-size: 14px;
+	line-height: 1.5em;
+}
+
+h1 {
+	font-family: 'Lato', Helvetica, Arial, Lucida, sans-serif;
+	font-size: 40px;
+	line-height: 45px;
+	margin-top: 45px ;
+	margin-bottom: 0px;
+	color: #003399 !important;
+}
+
+h2 {
+	font-family: 'Lato', Helvetica, Arial, Lucida, sans-serif;
+	font-size: 28px;
+	margin-bottom: 10px;
+	color: #003399 !important;
+	border-bottom: 1px solid #e1e1e1;
+}
+
+h3 {
+	font-family: 'Lato', Helvetica, Arial, Lucida, sans-serif;
+	font-size: 26px;
+	margin-bottom: 10px;
+	color: #003399 !important;
+	font-variant-caps: all-small-caps;
+}
+
+h4 {
+	font-family: 'Lato', Helvetica, Arial, Lucida, sans-serif;
+	font-size: 22px;
+	margin-bottom: 10px;
+	color: #003399 !important;
+	font-variant-caps: all-small-caps;
+}
+
+h2::before {
+    content: ">";
+    color: #e0e0eb;
+}
+
+#site-navigation h1.site-logo {
+    border-bottom: none;
+}
+
+#site-navigation {
+    background-color: #eeeeee;
+}
+
+.bd-sidebar .nav {
+    font-family: Verdana, Geneva, sans-serif;
+    font-size: 14px;
+    color: #555 !important;
+}

streamsets-6.3/docs/_static/full-streamsets-logo.svg

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="utf-8"?><svg xmlns="http://www.w3.org/2000/svg" width="207" height="44" viewBox="0 0 353.96 74.46"><defs><style>.cls-1{fill:#29b4e2;}.cls-2{fill:#039;}</style></defs><title>streamsets-logo</title><g id="Layer_2" data-name="Layer 2"><g id="Layer_2-2" data-name="Layer 2"><path class="cls-1" d="M111.7,45.77A9.38,9.38,0,0,1,108,53.64q-3.66,2.82-9.93,2.82-6.8,0-10.46-1.75V50.42A27.48,27.48,0,0,0,92.78,52a27.08,27.08,0,0,0,5.48.58c3,0,5.2-.57,6.7-1.69a5.54,5.54,0,0,0,2.24-4.69,6.06,6.06,0,0,0-.79-3.26,7.56,7.56,0,0,0-2.67-2.34,38.46,38.46,0,0,0-5.68-2.43q-5.34-1.9-7.62-4.52a9.94,9.94,0,0,1-2.29-6.82,8.45,8.45,0,0,1,3.32-7,13.8,13.8,0,0,1,8.78-2.62,25.94,25.94,0,0,1,10.48,2.09l-1.38,3.87a23.81,23.81,0,0,0-9.2-2,8.9,8.9,0,0,0-5.52,1.51,5,5,0,0,0-2,4.21,6.45,6.45,0,0,0,.74,3.26,6.8,6.8,0,0,0,2.47,2.32,30.21,30.21,0,0,0,5.31,2.34q6,2.15,8.28,4.6A9.05,9.05,0,0,1,111.7,45.77Z"/><path class="cls-1" d="M126.78,52.88a14.68,14.68,0,0,0,2.22-.17,14.5,14.5,0,0,0,1.7-.35v3.32a8.36,8.36,0,0,1-2.08.56,15.43,15.43,0,0,1-2.47.22q-8.31,0-8.31-8.76v-17h-4.11V28.57l4.11-1.8,1.83-6.12h2.51v6.64h8.31v3.37h-8.31V47.52a5.77,5.77,0,0,0,1.22,4A4.27,4.27,0,0,0,126.78,52.88Z"/><path class="cls-1" d="M148.15,26.77a17.07,17.07,0,0,1,3.43.31l-.61,4a14.25,14.25,0,0,0-3.13-.4,7.63,7.63,0,0,0-6,2.83,10.26,10.26,0,0,0-2.47,7V55.94h-4.34V27.29h3.58l.5,5.31h.21a12.51,12.51,0,0,1,3.84-4.32A8.66,8.66,0,0,1,148.15,26.77Z"/><path class="cls-1" d="M167.7,56.46q-6.36,0-10-3.87T154,41.85q0-6.93,3.41-11a11.36,11.36,0,0,1,9.16-4.07,10.8,10.8,0,0,1,8.52,3.54,13.52,13.52,0,0,1,3.14,9.34V42.4H158.5q.12,5,2.54,7.66a8.85,8.85,0,0,0,6.81,2.61A22.87,22.87,0,0,0,177,50.73V54.6A22.49,22.49,0,0,1,172.65,56,24.3,24.3,0,0,1,167.7,56.46ZM166.52,30.4a7.08,7.08,0,0,0-5.5,2.25,10.32,10.32,0,0,0-2.42,6.22h15a9.55,9.55,0,0,0-1.83-6.29A6.45,6.45,0,0,0,166.52,30.4Z"/><path class="cls-1" d="M201.91,55.94l-.86-4.08h-.21a12.19,12.19,0,0,1-4.28,3.65,13.12,13.12,0,0,1-5.32,1,9.56,9.56,0,0,1-6.68-2.2A8,8,0,0,1,182.15,48q0-8.69,13.88-9.1l4.86-.16V37q0-3.37-1.45-5a6,6,0,0,0-4.64-1.61,18.86,18.86,0,0,0-8.11,2.2l-1.33-3.32a20.7,20.7,0,0,1,9.7-2.46c3.41,0,5.95.76,7.59,2.27s2.47,3.95,2.47,7.29V55.94Zm-9.8-3.06a8.8,8.8,0,0,0,6.36-2.22,8.23,8.23,0,0,0,2.31-6.22V41.85l-4.34.18q-5.16.18-7.46,1.61a4.86,4.86,0,0,0-2.29,4.43,4.47,4.47,0,0,0,1.43,3.58A5.92,5.92,0,0,0,192.11,52.88Z"/><path class="cls-1" d="M249.24,55.94V37.3a7.8,7.8,0,0,0-1.46-5.14,5.68,5.68,0,0,0-4.55-1.71q-4,0-6,2.33t-1.93,7.16v16H231V37.3a7.8,7.8,0,0,0-1.46-5.14,5.72,5.72,0,0,0-4.58-1.71q-4.08,0-6,2.45c-1.26,1.63-1.9,4.3-1.9,8v15h-4.34V27.29h3.53l.71,3.92h.21a8.6,8.6,0,0,1,3.46-3.27,10.64,10.64,0,0,1,5-1.17q6.72,0,8.78,4.86h.21a9.17,9.17,0,0,1,3.71-3.56,11.53,11.53,0,0,1,5.54-1.3q4.86,0,7.28,2.49t2.42,8V55.94Z"/><path class="cls-2" d="M283.4,45.77a9.38,9.38,0,0,1-3.66,7.87q-3.66,2.82-9.94,2.82-6.8,0-10.45-1.75V50.42A27.61,27.61,0,0,0,264.47,52a27.25,27.25,0,0,0,5.49.58c3,0,5.19-.57,6.69-1.69a5.53,5.53,0,0,0,2.25-4.69,6,6,0,0,0-.8-3.26,7.52,7.52,0,0,0-2.66-2.34,38.2,38.2,0,0,0-5.69-2.43q-5.33-1.9-7.62-4.52a10,10,0,0,1-2.28-6.82,8.47,8.47,0,0,1,3.31-7A13.85,13.85,0,0,1,272,17.17a25.94,25.94,0,0,1,10.48,2.09L281,23.13a23.73,23.73,0,0,0-9.2-2,8.89,8.89,0,0,0-5.51,1.51,5,5,0,0,0-2,4.21,6.45,6.45,0,0,0,.73,3.26,6.88,6.88,0,0,0,2.47,2.32,30.37,30.37,0,0,0,5.32,2.34q6,2.15,8.28,4.6A9.1,9.1,0,0,1,283.4,45.77Z"/><path class="cls-2" d="M301.32,56.46q-6.35,0-10-3.87t-3.67-10.74q0-6.93,3.41-11a11.35,11.35,0,0,1,9.16-4.07,10.8,10.8,0,0,1,8.52,3.54,13.56,13.56,0,0,1,3.14,9.34V42.4H292.12q.13,5,2.55,7.66a8.82,8.82,0,0,0,6.81,2.61,22.87,22.87,0,0,0,9.15-1.94V54.6A22.71,22.71,0,0,1,306.28,56,24.33,24.33,0,0,1,301.32,56.46ZM300.15,30.4a7.1,7.1,0,0,0-5.51,2.25,10.38,10.38,0,0,0-2.41,6.22h15a9.61,9.61,0,0,0-1.83-6.29A6.47,6.47,0,0,0,300.15,30.4Z"/><path class="cls-2" d="M327.17,52.88a14.58,14.58,0,0,0,2.22-.17,14.07,14.07,0,0,0,1.7-.35v3.32a8.24,8.24,0,0,1-2.08.56,15.33,15.33,0,0,1-2.47.22q-8.31,0-8.31-8.76v-17h-4.1V28.57l4.1-1.8,1.83-6.12h2.51v6.64h8.31v3.37h-8.31V47.52a5.77,5.77,0,0,0,1.23,4A4.25,4.25,0,0,0,327.17,52.88Z"/><path class="cls-2" d="M354,48.12a7.18,7.18,0,0,1-3,6.17q-3,2.18-8.37,2.17-5.7,0-8.89-1.8v-4a21.38,21.38,0,0,0,4.44,1.65,18.45,18.45,0,0,0,4.56.6,10.25,10.25,0,0,0,5.22-1.09,3.73,3.73,0,0,0,.38-6.16,22.63,22.63,0,0,0-5.66-2.82,29.86,29.86,0,0,1-5.68-2.6,7.72,7.72,0,0,1-2.51-2.52,6.55,6.55,0,0,1-.82-3.37,6.42,6.42,0,0,1,2.85-5.53c1.89-1.35,4.5-2,7.81-2a22.83,22.83,0,0,1,9.05,1.88l-1.55,3.53A20.64,20.64,0,0,0,344,30.4a9,9,0,0,0-4.66,1A3,3,0,0,0,337.78,34a3.19,3.19,0,0,0,.59,2,5.81,5.81,0,0,0,1.89,1.55,43,43,0,0,0,5,2.11q5.1,1.86,6.88,3.74A6.58,6.58,0,0,1,354,48.12Z"/><path class="cls-2" d="M20.48,4.78c2.17-1.14,9.59-4.06,18.11,5.66,5.6,6.38,9,6.74,10,4.57h0a2.17,2.17,0,0,0,.26-.64c.62-2.64-1.33-7.63-6.15-11.25C32.08-4.86,20.1,5,20.1,5l.38-.21Z"/><path class="cls-2" d="M13.7,70.76c-2.09-1.33-8.28-6.31-4.14-18.51,2.73-8,1.34-11.18-1-10.93h0a2.7,2.7,0,0,0-.68.09c-2.6.79-5.94,5-6.67,11C-.43,65.54,14.08,71,14.08,71l-.37-.23Z"/><path class="cls-2" d="M74.25,43.2c0,.11,0,.26,0,.43h0c-.1,2.46-1.28,10.35-14,12.87C52,58.14,50,60.9,51.32,62.83a3,3,0,0,0,.43.56c2,1.85,7.28,2.66,12.83.29C76.79,58.49,74.25,43.2,74.25,43.2Z"/><path class="cls-1" d="M30.57,24.16C48.39,37.67,50.5,38.51,54.24,37.27c4.82-1.59,8.08-8.85,4.82-18.91s-10-12.88-10-12.88.29.34.75.94a34.18,34.18,0,0,1,3.44,5c3.11,5.63,3.27,10.57-.75,12.91s-11.56-5.87-15.07-8.9C26.71,6.22,19.57,6.05,15,9.28,9,13.52,7.37,23.62,8,26.74,9.14,22.15,13.21,11,30.57,24.16Z"/><path class="cls-1" d="M25.45,52.34C28.24,30.15,27.92,27.9,25,25.28c-3.8-3.38-11.71-2.57-18.8,5.28S0,45.67,0,45.67s.15-.42.43-1.12a35.07,35.07,0,0,1,2.65-5.49c3.32-5.51,7.51-8.12,11.55-5.81s.69,13-.17,17.5c-2.65,13.9.78,20.17,5.86,22.52,6.69,3.09,16.23-.57,18.64-2.63C34.43,71.9,22.74,74,25.45,52.34Z"/><path class="cls-1" d="M52.09,68.66c-6.43-.12-10.8-2.45-10.81-7.11S52.15,54.48,56.52,53C69.88,48.31,73.59,42.2,73.09,36.62c-.66-7.33-8.61-13.77-11.6-14.82,3.37,3.32,11,12.43-9.07,20.88C31.81,51.36,30,52.77,29.22,56.62c-1,5,3.63,11.43,14,13.63S59.35,68,59.35,68l-1.19.19A35.54,35.54,0,0,1,52.09,68.66Z"/></g></g></svg>

{streamsets-6.2.0.dev0 → streamsets-6.3}/docs/api/sch_api.rst

@@ -164,6 +164,19 @@ Jobs
 .. autoclass:: streamsets.sdk.sch_models.RuntimeParameters
     :members:
 
+Job Sequences
+^^^^^^^^^^^^^
+.. autoclass:: streamsets.sdk.sch_models.JobSequence
+    :members:
+.. autoclass:: streamsets.sdk.sch_models.JobSequences
+    :members:
+.. autoclass:: streamsets.sdk.sch_models.JobSequenceBuilder
+    :members:
+.. autoclass:: streamsets.sdk.sch_models.JobSequenceHistoryLog
+    :members:
+.. autoclass:: streamsets.sdk.sch_models.Step
+    :members:
+
 Organizations
 ^^^^^^^^^^^^^
 .. autoclass:: streamsets.sdk.sch_models.Organization
@@ -223,25 +236,6 @@ ProvisioningAgents
 .. autoclass:: streamsets.sdk.sch_models.ProvisioningAgents
     :members:
 
-Reports
-^^^^^^^
-.. autoclass:: streamsets.sdk.sch_models.GenerateReportCommand
-    :members:
-.. autoclass:: streamsets.sdk.sch_models.Report
-    :members:
-.. autoclass:: streamsets.sdk.sch_models.Reports
-    :members:
-.. autoclass:: streamsets.sdk.sch_models.ReportDefinition
-    :members:
-.. autoclass:: streamsets.sdk.sch_models.ReportDefinitions
-    :members:
-.. autoclass:: streamsets.sdk.sch_models.ReportDefinitionBuilder
-    :members:
-.. autoclass:: streamsets.sdk.sch_models.ReportResource
-    :members:
-.. autoclass:: streamsets.sdk.sch_models.ReportResources
-    :members:
-
 Roles
 ^^^^^
 .. autoclass:: streamsets.sdk.sch_models.Roles

streamsets-6.3/docs/build/error_record_handling.rst

@@ -0,0 +1,99 @@
+Handling Error Records
+======================
+
+Handling Error Records in the Platform SDK is very similar to handling error records in the UI.
+
+For more information, you can view the `Streamsets Documentation for Error Handling <https://docs.streamsets.com/portal/platform-datacollector/latest/datacollector/UserGuide/Pipeline_Design/ErrorHandling.html>`_.
+
+.. hint::
+  All of the examples below have focused on stages for SDC pipelines, however :py:class:`streamsets.sdk.sch_models.SchStStage`
+  instances could be swapped into these examples for Transformer pipelines without issue.
+
+Handling Error Records in a Pipeline
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the UI, you can edit how error records are handled by clicking on ``Show Advanced Options`` and then ``Error Records``.
+
+.. image:: ../_static/images/build/error_records_main.png
+|
+
+The default action for when a pipeline produces error records is to discard the record.
+This behavior can be modified, like writing the error records to file or sending it to a MQTT broker.
+
+To do so in the UI, click on the options for Error Records and select the appropriate option.
+
+You can do this in the Platform SDK by using :py:meth:`streamsets.sdk.sch_models.PipelineBuilder.add_error_stage` as follows:
+
+.. code-block:: python
+
+    error_stage = pipeline_builder.add_error_stage('Write to File')
+
+.. note::
+  Having different stage libraries in your engine's deployment configuration may show additional options than what are available by default..
+  For example: using the AWS library for your deployment will give an option to write your error records to S3 instead.
+
+Configuring Error Record Handling for a Pipeline
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the UI, choosing an option other than ``Discard`` will provide additional configuration options under the ``Error Records`` tab, as seen below:
+
+.. image:: ../_static/images/build/error_records_configuration.png
+|
+
+You can do this in the Platform SDK by accessing the `configuration` property of :py:class:`streamsets.sdk.sch_models.SchSdcStage` instance that represents your error stage for the pipeline:
+
+.. code-block:: python
+
+    error_stage.configuration.directory = '/path/to/error/directory'
+
+.. hint::
+  When writing to a directory you must first create the directory in the deployment, documentation for it can be found at: `Streamsets Documentation to manage a Local FS <https://docs.streamsets.com/portal/platform-datacollector/latest/datacollector/UserGuide/Destinations/LocalFS.html>`_.
+
+Changing the Error Record Policy for a Pipeline
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the UI, you can change the Error Record Policy from the main Error Records Tab
+
+.. image:: ../_static/images/build/error_records_main.png
+|
+
+The default value for Error Records Policy in the UI is ``Original record as it was generated by the origin`` but can be changed to ``Record as it was seen by the stage that sent it to error stream``.
+
+.. hint::
+  The equivalent values you can supply in the Platform SDK for Error Record Policy are ``ORIGINAL_RECORD`` and ``STAGE_RECORD``.
+
+To do this in the Platform SDK you can do the following:
+
+.. code-block:: python
+
+    # store the record as seen by the first stage in the pipeline
+    pipeline.error_record_policy = 'ORIGINAL_RECORD'
+
+    # store the record as seen by the stage that raised the error
+    pipeline.error_record_policy = 'STAGE_RECORD'
+
+Bringing It All Together
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+The complete scripts from this section can be found below. Commands that only served to verify some output from the example have been removed.
+
+.. code-block:: python
+
+    # creating a pipeline builder and adding stages
+    pipeline_builder = sch.get_pipeline_builder(...)
+    stage_1 = pipeline_builder.add_stage(...)
+
+    # adding an appropriate error stage for our use case, can leave it as is for the error records to be discarded
+    error_stage = pipeline_builder.add_error_stage('Write to File')
+
+    # remember to configure the error stage appropriately
+    error_stage.configuration.directory = '/path/to/error/directory'
+
+    # build the pipeline
+    pipeline = pipeline_builder.build('Pipeline name')
+
+    # you can, optionally, change the error records policy
+    pipeline.configuration.error_record_policy = 'STAGE_RECORD'
+
+    # now, the pipeline is ready to be published
+    sch.publish_pipeline(pipeline)

{streamsets-6.2.0.dev0 → streamsets-6.3}/docs/build.rst

@@ -11,3 +11,4 @@ Build
    build/creating_pipelines
    build/editing_pipelines
    build/pipeline_fragments
+   build/error_record_handling

{streamsets-6.2.0.dev0 → streamsets-6.3}/docs/monitor.rst

@@ -10,4 +10,3 @@ Monitor
 
    monitor/subscriptions
    monitor/topologies
-   monitor/reports

streamsets-6.3/docs/requirements.txt

@@ -0,0 +1,9 @@
+Sphinx~=4.4
+sphinx-book-theme~=0.3.0
+sphinx-copybutton~=0.5.0
+sphinxcontrib.applehelp==1.0.4
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==2.0.0
+sphinxcontrib-serializinghtml==1.1.5
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-jsmath==1.0.1