dkist-processing-core 4.0.0__tar.gz → 4.2.0__tar.gz

{dkist-processing-core-4.0.0 → dkist-processing-core-4.2.0}/CHANGELOG.rst

@@ -1,3 +1,26 @@
+v4.2.0 (2024-09-27)
+===================
+
+Misc
+----
+
+- Fixing deprecation warnings in pkg_resources. (`#39 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/39>`__)
+- Utility for generating the name of a workflow is part of the public API. (`#40 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/40>`__)
+- Upgrade to airflow 2.10.2. (`#41 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/41>`__)
+
+
+v4.1.0 (2024-07-01)
+===================
+
+Misc
+----
+
+- Add utility for generating the name of a workflow. (`#35 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/35>`__)
+- Update the instructions for development to include the dependency on rabbitmq and docker. (`#36 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/36>`__)
+- Make private methods public when we want them to show up in the ReadTheDocs documentation. (`#37 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/37>`__)
+- Upgrade airflow to version 2.9.2. (`#38 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/38>`__)
+
+
 v4.0.0 (2024-06-03)
 ===================
 

{dkist-processing-core-4.0.0 → dkist-processing-core-4.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-core
-Version: 4.0.0
+Version: 4.2.0
 Summary: Abstraction layer that is used by the DKIST Science Data Processing pipelines to process DKIST data using Apache Airflow.
 Home-page: https://bitbucket.org/dkistdc/dkist-processing-core/src/main/
 Author: NSO / AURA
@@ -150,12 +150,19 @@ Environment Variables
 
 Development
 -----------
+A prerequisite for test execution is a running instance of rabbitmq and docker on the local machine.
+For RabbitMQ the tests will use the default guest/guest credentials and a host ip of 127.0.0.1 and port of 5672 to connect to the broker.
+Getting docker set up varies by system, but the tests will use the default unix socket for the docker daemon.
+
+To run the tests locally, clone the repository and install the package in editable mode with the test extras.
+
 .. code-block:: bash
 
     git clone git@bitbucket.org:dkistdc/dkist-processing-core.git
     cd dkist-processing-core
     pre-commit install
     pip install -e .[test]
+    # RabbitMQ and Docker needs to be running
     pytest -v --cov dkist_processing_core
 
 Changelog

{dkist-processing-core-4.0.0 → dkist-processing-core-4.2.0}/README.rst

@@ -134,12 +134,19 @@ Environment Variables
 
 Development
 -----------
+A prerequisite for test execution is a running instance of rabbitmq and docker on the local machine.
+For RabbitMQ the tests will use the default guest/guest credentials and a host ip of 127.0.0.1 and port of 5672 to connect to the broker.
+Getting docker set up varies by system, but the tests will use the default unix socket for the docker daemon.
+
+To run the tests locally, clone the repository and install the package in editable mode with the test extras.
+
 .. code-block:: bash
 
     git clone git@bitbucket.org:dkistdc/dkist-processing-core.git
     cd dkist-processing-core
     pre-commit install
     pip install -e .[test]
+    # RabbitMQ and Docker needs to be running
     pytest -v --cov dkist_processing_core
 
 Changelog

{dkist-processing-core-4.0.0 → dkist-processing-core-4.2.0}/dkist_processing_core/__init__.py

@@ -1,13 +1,13 @@
 """Package-level setup information."""
-from pkg_resources import DistributionNotFound
-from pkg_resources import get_distribution
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version
 
 from dkist_processing_core.resource_queue import ResourceQueue
 from dkist_processing_core.task import TaskBase
 from dkist_processing_core.workflow import Workflow
 
 try:
-    __version__ = get_distribution(__name__).version
-except DistributionNotFound:
+    __version__ = version(distribution_name=__name__)
+except PackageNotFoundError:
     # package is not installed
     __version__ = "unknown"

{dkist-processing-core-4.0.0 → dkist-processing-core-4.2.0}/dkist_processing_core/build_utils.py

@@ -16,7 +16,7 @@ def validate_workflows(workflow_package: ModuleType, export_path: Path | None =
     rm_export_path_after_test = not bool(export_path)
     if export_path is None:
         export_path = Path("export/")
-    workflows = _extract_workflows_from_package(workflow_package)
+    workflows = extract_workflows_from_package(workflow_package)
     try:
         _validate_workflows(workflows, export_path)
     finally:
@@ -24,21 +24,31 @@ def validate_workflows(workflow_package: ModuleType, export_path: Path | None =
             rmtree(export_path)
 
 
+def _validate_workflows(workflows: list[Workflow], export_path: Path) -> None:
+    """Validate workflows by ensuring their exported version compiles as python and that there is at least one node."""
+    for w in workflows:
+        workflow_py = w.export_dag(path=export_path)
+        with workflow_py.open(mode="r") as f:
+            compile(f.read(), filename=f"{workflow_py.stem}.pyc", mode="exec")
+        if len(w.nodes) == 0:
+            raise ValueError(f"Workflow {w.workflow_name} has 0 nodes.")
+
+
 def export_dags(workflow_package: ModuleType, path: str | Path) -> list[Path]:
     """Export Airflow DAG files."""
-    return [w.export_dag(path=path) for w in _extract_workflows_from_package(workflow_package)]
+    return [w.export_dag(path=path) for w in extract_workflows_from_package(workflow_package)]
 
 
 def export_notebooks(workflow_package: ModuleType, path: str | Path) -> list[Path]:
     """Export Jupyter Notebook files."""
-    return [w.export_notebook(path=path) for w in _extract_workflows_from_package(workflow_package)]
+    return [w.export_notebook(path=path) for w in extract_workflows_from_package(workflow_package)]
 
 
 def export_notebook_dockerfile(workflow_package: ModuleType, path: str | Path) -> Path:
     """Export a dockerfile to containerize notebooks."""
     path = Path(path)
     notebook_paths = export_notebooks(workflow_package=workflow_package, path=path)
-    category = _extract_category_from_workflows(workflow_package=workflow_package)
+    category = extract_category_from_workflows(workflow_package=workflow_package)
     dockerfile = NotebookDockerfile(notebook_paths=notebook_paths, category=category)
     dockerfile_path = Path("Dockerfile")
     dockerfile_path.touch(exist_ok=False)
@@ -47,8 +57,9 @@ def export_notebook_dockerfile(workflow_package: ModuleType, path: str | Path) -
     return dockerfile_path
 
 
-def _extract_category_from_workflows(workflow_package: ModuleType) -> str:
-    workflows = _extract_workflows_from_package(workflow_package)
+def extract_category_from_workflows(workflow_package: ModuleType) -> str:
+    """Extract the category from the workflows in the package to provide a unique category for the dockerfile."""
+    workflows = extract_workflows_from_package(workflow_package)
     categories = {w.category for w in workflows}
     if len(categories) > 1:
         raise ValueError(
@@ -57,12 +68,14 @@ def _extract_category_from_workflows(workflow_package: ModuleType) -> str:
     return categories.pop()
 
 
-def _extract_workflows_from_package(workflow_package: ModuleType) -> list[Workflow]:
-    return _extract_objects_from_package_by_type(workflow_package, Workflow)
+def extract_workflows_from_package(workflow_package: ModuleType) -> list[Workflow]:
+    """Extract all the Workflow objects from a package."""
+    return extract_objects_from_package_by_type(workflow_package, Workflow)
 
 
-def _extract_objects_from_package_by_type(package: ModuleType, object_type: type) -> list:
-    modules = _parse_unprotected_modules_names_from_package(package)
+def extract_objects_from_package_by_type(package: ModuleType, object_type: type) -> list:
+    """Extract all objects in public modules of a given type from a package."""
+    modules = parse_unprotected_modules_names_from_package(package)
     objects = []
     for module in modules:
         imported_module = importlib.import_module(f".{module}", package.__name__)
@@ -70,21 +83,12 @@ def _extract_objects_from_package_by_type(package: ModuleType, object_type: type
     return objects
 
 
-def _parse_unprotected_modules_names_from_package(package: ModuleType) -> list[str]:
+def parse_unprotected_modules_names_from_package(package: ModuleType) -> list[str]:
+    """Parse the names of all modules in a package that are not private i.e. don't begin with an underscore."""
     package_path = Path(package.__path__[0])
     return [m.stem for m in package_path.glob("[!_]*.py")]
 
 
-def _validate_workflows(workflows: list[Workflow], export_path: Path) -> None:
-    """Validate workflows by ensuring their exported version compiles as python and that there is at least one node."""
-    for w in workflows:
-        workflow_py = w.export_dag(path=export_path)
-        with workflow_py.open(mode="r") as f:
-            compile(f.read(), filename=f"{workflow_py.stem}.pyc", mode="exec")
-        if len(w.nodes) == 0:
-            raise ValueError(f"Workflow {w.workflow_name} has 0 nodes.")
-
-
 class NotebookDockerfile:
     """Build a Dockerfile for deployment as a Manual Processing Worker."""
 

dkist-processing-core-4.0.0/dkist_processing_core/_node.py → dkist-processing-core-4.2.0/dkist_processing_core/node.py

@@ -50,7 +50,7 @@ class Node:
     def operator(self) -> BashOperator:
         """Native engine node."""
         from datetime import timedelta
-        from dkist_processing_core._failure_callback import chat_ops_notification
+        from dkist_processing_core.failure_callback import chat_ops_notification
         from functools import partial
 
         return eval(self.operator_definition)
@@ -69,7 +69,7 @@ class Node:
         """Airflow style command to define a bash operator."""
         return f"""BashOperator(
     task_id='{self.task.__name__}',
-    bash_command='''{self._bash_script}''',
+    bash_command='''{self.bash_script}''',
     retries={self.task.retries},
     retry_delay=timedelta(seconds={self.task.retry_delay_seconds}),
     on_failure_callback=partial(
@@ -89,14 +89,15 @@ class Node:
         return [(upstream.__name__, self.task.__name__) for upstream in self.upstreams]
 
     @property
-    def _bash_script(self) -> str:
+    def bash_script(self) -> str:
         """Format bash script for the BashOperator."""
-        command = f"""{self._install_command}
-{self._run_command}"""
-        return self._bash_template(command)
+        command = f"""{self.install_command}
+{self.run_command}"""
+        return self.bash_template(command)
 
     @staticmethod
-    def _bash_template(command: str) -> str:
+    def bash_template(command: str) -> str:
+        """Return the bash script with a template wrapped command."""
         return f"""#!/bin/bash
 echo Working Directory
 pwd
@@ -129,26 +130,30 @@ echo Exit with code from main command: $exit_code
 exit $exit_code"""
 
     @property
-    def _formatted_pip_extras(self) -> str:
+    def formatted_pip_extras(self) -> str:
+        """Format pip extras for the installation command."""
         if self.pip_extras:
             extra_requirements = ",".join(self.pip_extras)
             return f"'[{extra_requirements}]'"
         return ""
 
     @property
-    def _install_command(self) -> str:
+    def install_command(self) -> str:
+        """Format the installation command for the bash script."""
         repo_name = self.workflow_package.split(".")[0].replace("_", "-")
         version = self.workflow_version
-        extras = self._formatted_pip_extras
+        extras = self.formatted_pip_extras
         return f"""python -m pip install --upgrade pip
 python -m pip install {repo_name}{extras}=={version}"""
 
     @property
-    def _run_command(self) -> str:
-        return f'python -c "{self._python}"'
+    def run_command(self) -> str:
+        """Return the python bash command to execute the task."""
+        return f'python -c "{self.python}"'
 
     @property
-    def _python(self) -> str:
+    def python(self) -> str:
+        """Return the python code to execute the task."""
         return f"""from {self.task.__module__} import {self.task.__name__}
 with {self.task.__name__}(recipe_run_id={{{{dag_run.conf['recipe_run_id']}}}}, workflow_name='{self.workflow_name}', workflow_version='{self.workflow_version}') as task:
     task()

{dkist-processing-core-4.0.0 → dkist-processing-core-4.2.0}/dkist_processing_core/task.py

@@ -27,17 +27,18 @@ class ApmTransaction:
     """
 
     @property
-    def _apm_service_name(self) -> str:
+    def apm_service_name(self) -> str:
+        """Format the service name for Elastic APM."""
         name = f"{self._workflow_name}-{self._workflow_version}"
         name = name.replace("_", "-")
         name = name.replace(".", "-")
         return name
 
     @property
-    def _apm_config(self) -> dict:
+    def apm_config(self) -> dict:
+        """Override the Elastic APM configuration with the workflow specific service name."""
         core_config = core_configurations.apm_config
-        #  Override the service name with the workflow specific name
-        core_config["SERVICE_NAME"] = self._apm_service_name
+        core_config["SERVICE_NAME"] = self.apm_service_name
         return core_config
 
     def __init__(self, transaction_name: str, workflow_name: str, workflow_version: str) -> None:
@@ -46,10 +47,10 @@ class ApmTransaction:
         self.transaction_name = transaction_name
 
         if core_configurations.elastic_apm_enabled:
-            self.client = elasticapm.Client(self._apm_config)
+            self.client = elasticapm.Client(self.apm_config)
             self.instrument()
             self.client.begin_transaction(transaction_type="Task")
-            logger.info(f"APM Configured: {self=} {self._apm_config=}")
+            logger.info(f"APM Configured: {self=} {self.apm_config=}")
         else:
             logger.warning(f"APM Not Configured")
 

{dkist-processing-core-4.0.0 → dkist-processing-core-4.2.0}/dkist_processing_core/tests/conftest.py

@@ -11,8 +11,8 @@ from talus import DurableProducer
 from dkist_processing_core import ResourceQueue
 from dkist_processing_core import TaskBase
 from dkist_processing_core import Workflow
-from dkist_processing_core._node import Node
-from dkist_processing_core._node import task_type_hint
+from dkist_processing_core.node import Node
+from dkist_processing_core.node import task_type_hint
 from dkist_processing_core.tests.task_example import Task
 
 

{dkist-processing-core-4.0.0 → dkist-processing-core-4.2.0}/dkist_processing_core/tests/test_failure_callback.py

@@ -2,11 +2,11 @@
 import pytest
 from talus import DurableProducer
 
-from dkist_processing_core._failure_callback import chat_ops_notification
-from dkist_processing_core._failure_callback import parse_dag_run_id_from_context
-from dkist_processing_core._failure_callback import parse_log_url_from_context
-from dkist_processing_core._failure_callback import recipe_run_failure_message_producer_factory
-from dkist_processing_core._failure_callback import RecipeRunFailureMessage
+from dkist_processing_core.failure_callback import chat_ops_notification
+from dkist_processing_core.failure_callback import parse_dag_run_id_from_context
+from dkist_processing_core.failure_callback import parse_log_url_from_context
+from dkist_processing_core.failure_callback import recipe_run_failure_message_producer_factory
+from dkist_processing_core.failure_callback import RecipeRunFailureMessage
 
 
 @pytest.fixture()

{dkist-processing-core-4.0.0 → dkist-processing-core-4.2.0}/dkist_processing_core/tests/test_node.py

@@ -1,4 +1,4 @@
-"""Tests of the _node.py module."""
+"""Tests of the node.py module."""
 import subprocess
 from subprocess import CalledProcessError
 from typing import Callable
@@ -8,7 +8,7 @@ from airflow.operators.bash import BashOperator
 from jinja2 import Template
 
 from dkist_processing_core import ResourceQueue
-from dkist_processing_core._node import Node
+from dkist_processing_core.node import Node
 from dkist_processing_core.tests.task_example import Task
 
 
@@ -38,8 +38,8 @@ def test_nodes(node, fake_producer_factory, queue_name, pip_extras):
     # passing in just a context dict positional arg with a fake http adapter does not raise an error
     failure_callback_func({"context": True}, producer_factory=fake_producer_factory)
     assert isinstance(operator, BashOperator)
-    assert node._install_command in operator.bash_command
-    assert node._python in operator.bash_command
+    assert node.install_command in operator.bash_command
+    assert node.python in operator.bash_command
     assert node.workflow_name == name
     assert node.upstreams == upstream
     assert node.task == task
@@ -57,7 +57,7 @@ def test_node_bash_template_return_0(node):
     """
     node, *args = node
     cmd = 'python -c "pass"'
-    result = subprocess.run(node._bash_template(cmd), shell=True, check=True)
+    result = subprocess.run(node.bash_template(cmd), shell=True, check=True)
     assert result.returncode == 0
 
 
@@ -71,7 +71,7 @@ def test_node_bash_template_return_1(node):
     node, *args = node
     cmd = 'python -c "raise Exception"'
     with pytest.raises(CalledProcessError):
-        subprocess.run(node._bash_template(cmd), shell=True, check=True)
+        subprocess.run(node.bash_template(cmd), shell=True, check=True)
 
 
 def test_node_python(single_node):
@@ -81,7 +81,7 @@ def test_node_python(single_node):
     Then: no exceptions raised.
     """
     # Given
-    code_template = Template(single_node._python)
+    code_template = Template(single_node.python)
 
     class RenderData:
         def __init__(self):

{dkist-processing-core-4.0.0 → dkist-processing-core-4.2.0}/dkist_processing_core/tests/test_workflow.py

@@ -8,6 +8,7 @@ from airflow import DAG
 from dkist_processing_core import ResourceQueue
 from dkist_processing_core import Workflow
 from dkist_processing_core.workflow import MAXIMUM_ALLOWED_WORKFLOW_NAME_LENGTH
+from dkist_processing_core.workflow import workflow_name_from_details
 
 
 def test_workflow_metadata(workflow):
@@ -39,7 +40,7 @@ def test_workflow_metadata(workflow):
     assert workflow_instance.input_data == input_data
     assert workflow_instance.output_data == output_data
     assert workflow_instance.detail == detail
-    assert sorted(json.loads(workflow_instance._dag_tags)) == sorted(
+    assert sorted(json.loads(workflow_instance.dag_tags)) == sorted(
         [tag for tag in tags] + [input_data, output_data, category, version]
     )
 
@@ -166,3 +167,46 @@ def test_check_dag_name_characters():
     Workflow.check_dag_name_characters(dag_name="This_dag_name_is_valid")
     with pytest.raises(ValueError):
         Workflow.check_dag_name_characters(dag_name="Invalid*dag*name")
+
+
+@pytest.mark.parametrize(
+    "detail",
+    [
+        pytest.param(None, id="no_detail"),
+        pytest.param("detail", id="with_detail"),
+    ],
+)
+def test_workflow_name_from_details(detail: str | None):
+    """
+    Given: a set of details
+    When: creating a workflow name
+    Then: the workflow name is created correctly
+    """
+    input_data = "input"
+    output_data = "output"
+    category = "instrument"
+    expected_workflow_name = f"{input_data}_to_{output_data}_{category}"
+    if detail:
+        expected_workflow_name += f"_{detail}"
+    workflow_name = workflow_name_from_details(
+        input_data=input_data,
+        output_data=output_data,
+        category=category,
+        detail=detail,
+    )
+    assert workflow_name == expected_workflow_name
+
+
+def test_workflow_name_from_details_too_long():
+    """
+    Given: workflow details with a long input_data value
+    When: calling workflow_name_from_details
+    Then: a ValueError is raised
+    """
+
+    with pytest.raises(ValueError):
+        workflow_name = workflow_name_from_details(
+            input_data="".join(["a" for _ in range(MAXIMUM_ALLOWED_WORKFLOW_NAME_LENGTH)]),
+            output_data="",
+            category="",
+        )

{dkist-processing-core-4.0.0 → dkist-processing-core-4.2.0}/dkist_processing_core/workflow.py

@@ -8,16 +8,32 @@ import nbformat as nbf
 from airflow import DAG
 
 from dkist_processing_core import ResourceQueue
-from dkist_processing_core._node import Node
-from dkist_processing_core._node import task_type_hint
-from dkist_processing_core._node import upstreams_type_hint
 from dkist_processing_core.config import core_configurations
+from dkist_processing_core.node import Node
+from dkist_processing_core.node import task_type_hint
+from dkist_processing_core.node import upstreams_type_hint
 
-__all__ = ["Workflow"]
+__all__ = ["Workflow", "workflow_name_from_details"]
 
 MAXIMUM_ALLOWED_WORKFLOW_NAME_LENGTH = 100
 
 
+def workflow_name_from_details(
+    input_data: str, output_data: str, category: str, detail: str | None = None
+) -> str:
+    """Create the workflow name from its constituent parts."""
+    workflow_name = f"{input_data}_to_{output_data}_{category}"
+    if detail:
+        workflow_name += f"_{detail}"
+    workflow_name_too_long = len(workflow_name) > MAXIMUM_ALLOWED_WORKFLOW_NAME_LENGTH
+    if workflow_name_too_long:
+        raise ValueError(
+            f"Workflow name {workflow_name} is {len(workflow_name)} characters long. "
+            f"Limit is {MAXIMUM_ALLOWED_WORKFLOW_NAME_LENGTH} characters."
+        )
+    return workflow_name
+
+
 class Workflow:
     """
     Abstraction to create a workflow in 1 or more target execution environment.
@@ -84,25 +100,21 @@ class Workflow:
         if isinstance(tags, str):
             tags = [tags]
         self.tags: list[str] = tags or []
-        self._dag = self._initialize_local_dag()
+        self._dag = self.initialize_local_dag()
         self.nodes = []
 
     @property
     def workflow_name(self) -> str:
         """Return the workflow name created from its constituent parts."""
-        result = f"{self.input_data}_to_{self.output_data}_{self.category}"
-        if self.detail:
-            result += f"_{self.detail}"
-        workflow_name_too_long = len(result) > MAXIMUM_ALLOWED_WORKFLOW_NAME_LENGTH
-        if workflow_name_too_long:
-            raise ValueError(
-                f"Workflow name {result} is {len(result)} characters long. "
-                f"Limit is {MAXIMUM_ALLOWED_WORKFLOW_NAME_LENGTH} characters."
-            )
-        return result
+        return workflow_name_from_details(
+            input_data=self.input_data,
+            output_data=self.output_data,
+            category=self.category,
+            detail=self.detail,
+        )
 
     @property
-    def _dag_name(self) -> str:
+    def dag_name(self) -> str:
         """Return the dag name created from its constituent parts."""
         result = f"{self.workflow_name}_{self.workflow_version}"
         self.check_dag_name_characters(result)  # raise an error if in valid
@@ -133,7 +145,7 @@ class Workflow:
             )
 
     @property
-    def _dag_tags(self) -> str:
+    def dag_tags(self) -> str:
         """
         Return the list of dag tags to be used in Airflow's UI.
 
@@ -146,14 +158,15 @@ class Workflow:
         return json.dumps(tags)
 
     @property
-    def _dag_definition(self) -> str:
-
-        return f"DAG(dag_id='{self._dag_name}', start_date=pendulum.today('UTC').add(days=-2), schedule=None, catchup=False, tags={self._dag_tags})"
+    def dag_definition(self) -> str:
+        """Return the string representation of the DAG object instantiation."""
+        return f"DAG(dag_id='{self.dag_name}', start_date=pendulum.today('UTC').add(days=-2), schedule=None, catchup=False, tags={self.dag_tags})"
 
-    def _initialize_local_dag(self) -> DAG:
+    def initialize_local_dag(self) -> DAG:
+        """Create a local instance of the DAG object."""
         import pendulum
 
-        return eval(self._dag_definition)
+        return eval(self.dag_definition)
 
     def add_node(
         self,
@@ -190,22 +203,22 @@ class Workflow:
         path = path or "dags/"
         path = Path(path)
         path.mkdir(exist_ok=True)
-        workflow_py = path / f"{self._dag_name}.py"
+        workflow_py = path / f"{self.dag_name}.py"
 
         with workflow_py.open(mode="w") as f:
             f.write(
                 f"#  {self.workflow_name} workflow version {self.workflow_version} definition rendered for airflow scheduler\n"
             )
-            f.write(self._workflow_imports)
+            f.write(self.workflow_imports)
             f.write("# Workflow\n")
-            f.write(self._workflow_instantiation)
+            f.write(self.workflow_instantiation)
             f.write("    # Nodes\n")
             for n in self.nodes:
                 operator = f"{n.task.__name__.lower()}_operator"
                 f.write(f"    {operator} = {n.operator_definition}")
                 f.write("\n")
             f.write("    # Edges\n")
-            f.write(self._workflow_edges)
+            f.write(self.workflow_edges)
             f.write("\n")
         return workflow_py
 
@@ -214,7 +227,7 @@ class Workflow:
         path = path or "notebooks/"
         path = Path(path)
         path.mkdir(exist_ok=True)
-        notebook_ipynb = path / f"{self._dag_name}.ipynb"
+        notebook_ipynb = path / f"{self.dag_name}.ipynb"
 
         nb = nbf.v4.new_notebook()
         nb["cells"].append(nbf.v4.new_code_cell("recipe_run_id: int ="))
@@ -233,7 +246,8 @@ class Workflow:
         return valid_node_order
 
     @property
-    def _workflow_imports(self) -> str:
+    def workflow_imports(self) -> str:
+        """Return the import statements for the workflow."""
         imports = [
             "from datetime import timedelta",
             "from functools import partial",
@@ -242,18 +256,20 @@ class Workflow:
             "from airflow.operators.bash import BashOperator",
             "import pendulum",
             "",
-            "from dkist_processing_core._failure_callback import chat_ops_notification",
+            "from dkist_processing_core.failure_callback import chat_ops_notification",
             "",
             "",
         ]
         return "\n".join(imports)
 
     @property
-    def _workflow_instantiation(self) -> str:
-        return f"with {self._dag_definition} as d:\n    pass\n"
+    def workflow_instantiation(self) -> str:
+        """Return the context manager instantiation of the workflow object."""
+        return f"with {self.dag_definition} as d:\n    pass\n"
 
     @property
-    def _workflow_edges(self) -> str:
+    def workflow_edges(self) -> str:
+        """Return the edges between nodes for the workflow."""
         edges = []
         for n in self.nodes:
             for upstream, downstream in n.dependencies:

{dkist-processing-core-4.0.0 → dkist-processing-core-4.2.0}/dkist_processing_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-core
-Version: 4.0.0
+Version: 4.2.0
 Summary: Abstraction layer that is used by the DKIST Science Data Processing pipelines to process DKIST data using Apache Airflow.
 Home-page: https://bitbucket.org/dkistdc/dkist-processing-core/src/main/
 Author: NSO / AURA
@@ -150,12 +150,19 @@ Environment Variables
 
 Development
 -----------
+A prerequisite for test execution is a running instance of rabbitmq and docker on the local machine.
+For RabbitMQ the tests will use the default guest/guest credentials and a host ip of 127.0.0.1 and port of 5672 to connect to the broker.
+Getting docker set up varies by system, but the tests will use the default unix socket for the docker daemon.
+
+To run the tests locally, clone the repository and install the package in editable mode with the test extras.
+
 .. code-block:: bash
 
     git clone git@bitbucket.org:dkistdc/dkist-processing-core.git
     cd dkist-processing-core
     pre-commit install
     pip install -e .[test]
+    # RabbitMQ and Docker needs to be running
     pytest -v --cov dkist_processing_core
 
 Changelog

{dkist-processing-core-4.0.0 → dkist-processing-core-4.2.0}/dkist_processing_core.egg-info/SOURCES.txt

@@ -10,10 +10,10 @@ setup.cfg
 setup.py
 changelog/.gitempty
 dkist_processing_core/__init__.py
-dkist_processing_core/_failure_callback.py
-dkist_processing_core/_node.py
 dkist_processing_core/build_utils.py
 dkist_processing_core/config.py
+dkist_processing_core/failure_callback.py
+dkist_processing_core/node.py
 dkist_processing_core/resource_queue.py
 dkist_processing_core/task.py
 dkist_processing_core/workflow.py

{dkist-processing-core-4.0.0 → dkist-processing-core-4.2.0}/dkist_processing_core.egg-info/requires.txt

@@ -1,4 +1,4 @@
-apache-airflow[celery,postgres]==2.9.1
+apache-airflow[celery,postgres]==2.10.2
 elastic-apm<7.0.0
 requests>=2.23
 talus<2.0,>=1.1.0
@@ -25,3 +25,4 @@ pytest-mock
 jinja2
 towncrier
 nbconvert
+ipython

{dkist-processing-core-4.0.0 → dkist-processing-core-4.2.0}/pyproject.toml

@@ -4,6 +4,7 @@
     directory = "changelog/"
     issue_format = "`#{issue} <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/{issue}>`__"
     title_format = "{version} ({project_date})"
+    ignore = [".gitempty"]
 
 [[tool.towncrier.type]]
       directory = "feature"

{dkist-processing-core-4.0.0 → dkist-processing-core-4.2.0}/setup.cfg

@@ -19,7 +19,7 @@ setup_requires = setuptools_scm
 packages = find:
 include_package_data = True
 install_requires = 
-	apache-airflow[postgres, celery] == 2.9.1
+	apache-airflow[postgres, celery] == 2.10.2
 	elastic-apm < 7.0.0
 	requests >= 2.23
 	talus >= 1.1.0, <2.0
@@ -36,6 +36,7 @@ test =
 	jinja2
 	towncrier
 	nbconvert
+	ipython
 docs = 
 	sphinx
 	sphinx-astropy