dkist-processing-core 2.0.2rc1__tar.gz → 2.1.0__tar.gz

{dkist-processing-core-2.0.2rc1 → dkist-processing-core-2.1.0}/CHANGELOG.rst

@@ -1,3 +1,21 @@
+v2.1.0 (2023-09-08)
+===================
+
+Features
+--------
+
+- Support assigning a resource queue to a node in a workflow when adding it to the workflow. (`#22 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/22>`__)
+
+
+v2.0.2 (2023-07-11)
+===================
+
+Misc
+----
+
+- Update airflow dependency to 2.6.3 (`#21 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/21>`__)
+
+
 v2.0.1 (2023-06-28)
 ===================
 

{dkist-processing-core-2.0.2rc1 → dkist-processing-core-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-core
-Version: 2.0.2rc1
+Version: 2.1.0
 Summary: Abstraction layer that is used by the DKIST Science Data Processing pipelines to process DKIST data using Apache Airflow.
 Home-page: https://bitbucket.org/dkistdc/dkist-processing-core/src/main/
 Author: NSO / AURA

{dkist-processing-core-2.0.2rc1 → dkist-processing-core-2.1.0}/dkist_processing_core/__init__.py

@@ -2,6 +2,7 @@
 from pkg_resources import DistributionNotFound
 from pkg_resources import get_distribution
 
+from dkist_processing_core.resource_queue import ResourceQueue
 from dkist_processing_core.task import TaskBase
 from dkist_processing_core.workflow import Workflow
 

{dkist-processing-core-2.0.2rc1 → dkist-processing-core-2.1.0}/dkist_processing_core/_node.py

@@ -4,6 +4,7 @@ from typing import Type
 
 from airflow.operators.bash import BashOperator
 
+from dkist_processing_core.resource_queue import ResourceQueue
 from dkist_processing_core.task import TaskBase
 
 
@@ -20,6 +21,7 @@ class Node:
         workflow_version: str,
         workflow_package: str,
         task: task_type_hint,
+        resource_queue: ResourceQueue,
         upstreams: upstreams_type_hint = None,
     ):
         """Node setup."""
@@ -40,6 +42,7 @@ class Node:
         self.task = task
         self.workflow_package = workflow_package
         self.upstreams = upstreams
+        self.resource_queue = resource_queue
 
     @property
     def operator(self) -> BashOperator:
@@ -65,6 +68,7 @@ class Node:
         task_name='{self.task.__name__}'
     ),
     owner="DKIST Data Center",
+    queue="{self.resource_queue.value}",
 )
 """
 
@@ -130,3 +134,11 @@ python -m pip install {repo_name}=={version}"""
 with {self.task.__name__}(recipe_run_id={{{{dag_run.conf['recipe_run_id']}}}}, workflow_name='{self.workflow_name}', workflow_version='{self.workflow_version}') as task:
     task()
 """
+
+    def __repr__(self):
+        """Render node instantiation as a string."""
+        return f"Node(workflow_name={self.workflow_name}, workflow_version={self.workflow_version}, workflow_package={self.workflow_package}, task={self.task!r}, upstreams={self.upstreams}, queue={self.resource_queue!r})"
+
+    def __str__(self):
+        """Render node instance as a string."""
+        return repr(self)

dkist-processing-core-2.1.0/dkist_processing_core/resource_queue.py

@@ -0,0 +1,9 @@
+"""Resource queue names for Workflow Node specification which specify different resource needs."""
+from enum import StrEnum
+
+
+class ResourceQueue(StrEnum):
+    """Supported queue names."""
+
+    DEFAULT: str = "default"
+    HIGH_MEMORY: str = "high_memory"

{dkist-processing-core-2.0.2rc1 → dkist-processing-core-2.1.0}/dkist_processing_core/tests/conftest.py

@@ -6,6 +6,7 @@ from typing import Any
 
 import pytest
 
+from dkist_processing_core import ResourceQueue
 from dkist_processing_core import TaskBase
 from dkist_processing_core import Workflow
 from dkist_processing_core._node import Node
@@ -104,8 +105,16 @@ def workflow_tasks(task_subclass) -> list[task_type_hint]:
     return [TaskA, TaskB, TaskC, TaskD]
 
 
+@pytest.fixture(params=["default", "non_default"])
+def queue_name(request):
+    """Name of the queue on the Node"""
+    if request.param == "default":
+        return ResourceQueue.DEFAULT
+    return ResourceQueue.HIGH_MEMORY
+
+
 @pytest.fixture(params=["0_upstream", "1_upstream", "2_upstream"])
-def node(workflow_tasks, request) -> tuple[Node, task_type_hint, Any, str, str]:
+def node(workflow_tasks, request, queue_name) -> tuple[Node, task_type_hint, Any, str, str]:
     """Node instance and its component parts."""
     version = "V6-123"
     name = f"{request.param}_{version}"
@@ -124,6 +133,7 @@ def node(workflow_tasks, request) -> tuple[Node, task_type_hint, Any, str, str]:
             workflow_package=package,
             task=TaskA,
             upstreams=upstream[0],
+            resource_queue=queue_name,
         ),
         TaskA,
         upstream[1],

{dkist-processing-core-2.0.2rc1 → dkist-processing-core-2.1.0}/dkist_processing_core/tests/test_failure_callback.py

@@ -46,10 +46,10 @@ def env_vars(request):
 def test_recipe_run_failure_producer_factory(env_vars):
     """
     Given: recipe_run_failure_producer_factory.
-    When: retrieving an producer from the factory.
+    When: retrieving a producer from the factory.
     Then: it is an instance of a talus.DurableBlockingProducerWrapper.
     """
-    with pytest.raises(pika.exceptions.AMQPConnectionError):
+    with pytest.raises(pika.exceptions.AMQPError):
         recipe_run_failure_message_producer_factory().__enter__()
 
 

{dkist-processing-core-2.0.2rc1 → dkist-processing-core-2.1.0}/dkist_processing_core/tests/test_node.py

@@ -1,16 +1,30 @@
 """Tests of the _node.py module."""
 import subprocess
 from subprocess import CalledProcessError
+from typing import Callable
 
 import pytest
 from airflow.operators.bash import BashOperator
 from jinja2 import Template
 
+from dkist_processing_core import ResourceQueue
 from dkist_processing_core._node import Node
 from dkist_processing_core.tests.task_example import Task
 
 
-def test_nodes(node, fake_producer_factory):
+@pytest.fixture()
+def single_node() -> Node:
+    """A single Node instance."""
+    return Node(
+        workflow_name="test_node",
+        workflow_version="v1",
+        workflow_package=__package__,
+        task=Task,
+        resource_queue=ResourceQueue.DEFAULT,
+    )
+
+
+def test_nodes(node, fake_producer_factory, queue_name):
     """
     Given: Workflow tasks to initialize a Node.
     When: Initializing the Node with valid task and upstreams.
@@ -29,6 +43,7 @@ def test_nodes(node, fake_producer_factory):
     assert node.upstreams == upstream
     assert node.task == task
     assert node.workflow_version == version
+    assert node.resource_queue == queue_name
 
 
 def test_node_bash_template_return_0(node):
@@ -55,20 +70,14 @@ def test_node_bash_template_return_1(node):
         subprocess.run(node._bash_template(cmd), shell=True, check=True)
 
 
-def test_node_python():
+def test_node_python(single_node):
     """
     Given: Python jinja rendered with dag run data from a node instance.
     When: parsing the python call.
     Then: no exceptions raised.
     """
     # Given
-    node = Node(
-        workflow_name="test_node_python",
-        workflow_version="v1",
-        workflow_package=__package__,
-        task=Task,
-    )
-    code_template = Template(node._python)
+    code_template = Template(single_node._python)
 
     class RenderData:
         def __init__(self):
@@ -114,3 +123,20 @@ def test_invalid_node(task_subclass):
             task=GoodTask,
             upstreams=[GoodTask, BadTask],
         )
+
+
+@pytest.mark.parametrize(
+    "func, attr",
+    [
+        pytest.param(repr, "__repr__", id="repr"),
+        pytest.param(str, "__str__", id="str"),
+    ],
+)
+def test_node_dunder(single_node, func: Callable, attr: str):
+    """
+    Given: Node instance
+    When: retrieving dunder method that should be implemented.
+    Then: It is implemented.
+    """
+    assert getattr(single_node, attr, None)
+    assert func(single_node)

{dkist-processing-core-2.0.2rc1 → dkist-processing-core-2.1.0}/dkist_processing_core/tests/test_workflow.py

@@ -5,6 +5,7 @@ from typing import Callable
 import pytest
 from airflow import DAG
 
+from dkist_processing_core import ResourceQueue
 from dkist_processing_core import Workflow
 
 
@@ -42,7 +43,14 @@ def test_workflow_metadata(workflow):
     )
 
 
-def test_workflow_add_node(workflow_tasks, workflow):
+@pytest.mark.parametrize(
+    "queue",
+    [
+        pytest.param(None, id="None"),
+        pytest.param(ResourceQueue.HIGH_MEMORY, id="Specified"),
+    ],
+)
+def test_workflow_add_node(workflow_tasks, workflow, queue):
     """
     Given: A set of tasks and a workflow instance.
     When: Adding the tasks to the workflow in the
@@ -79,7 +87,7 @@ def test_workflow_add_node(workflow_tasks, workflow):
         },
     }
     for task, upstream in task_definitions.items():
-        workflow_instance.add_node(task, upstreams=upstream)
+        workflow_instance.add_node(task, resource_queue=queue, upstreams=upstream)
 
     dag = workflow_instance.load()
     assert dag.task_count == 4
@@ -115,9 +123,9 @@ def test_invalid_workflow_add_node(workflow):
         pytest.param(str, "__str__", id="str"),
     ],
 )
-def test_tag_db_dunder(workflow, func: Callable, attr):
+def test_workflow_dunder(workflow, func: Callable, attr):
     """
-    Given: Connection to a tag database.
+    Given: workflow instance.
     When: retrieving dunder method that should be implemented.
     Then: It is implemented.
     """

{dkist-processing-core-2.0.2rc1 → dkist-processing-core-2.1.0}/dkist_processing_core/workflow.py

@@ -6,6 +6,7 @@ from pathlib import Path
 
 from airflow import DAG
 
+from dkist_processing_core import ResourceQueue
 from dkist_processing_core._node import Node
 from dkist_processing_core._node import task_type_hint
 from dkist_processing_core._node import upstreams_type_hint
@@ -161,13 +162,18 @@ class Workflow:
         self,
         task: task_type_hint,
         upstreams: upstreams_type_hint = None,
+        resource_queue: ResourceQueue | None = None,
     ) -> None:
         """Add a node and edges from that node to the workflow."""
+        if resource_queue is None:
+            resource_queue = ResourceQueue.DEFAULT
+
         node = Node(
             workflow_name=self.workflow_name,
             workflow_version=self.workflow_version,
             workflow_package=self.workflow_package,
             task=task,
+            resource_queue=resource_queue,
             upstreams=upstreams,
         )
         self.nodes.append(node)

{dkist-processing-core-2.0.2rc1 → dkist-processing-core-2.1.0}/dkist_processing_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-core
-Version: 2.0.2rc1
+Version: 2.1.0
 Summary: Abstraction layer that is used by the DKIST Science Data Processing pipelines to process DKIST data using Apache Airflow.
 Home-page: https://bitbucket.org/dkistdc/dkist-processing-core/src/main/
 Author: NSO / AURA

{dkist-processing-core-2.0.2rc1 → dkist-processing-core-2.1.0}/dkist_processing_core.egg-info/SOURCES.txt

@@ -9,11 +9,11 @@ pyproject.toml
 setup.cfg
 setup.py
 changelog/.gitempty
-changelog/21.misc.rst
 dkist_processing_core/__init__.py
 dkist_processing_core/_failure_callback.py
 dkist_processing_core/_node.py
 dkist_processing_core/build_utils.py
+dkist_processing_core/resource_queue.py
 dkist_processing_core/task.py
 dkist_processing_core/workflow.py
 dkist_processing_core.egg-info/PKG-INFO

dkist-processing-core-2.0.2rc1/changelog/21.misc.rst

@@ -1 +0,0 @@
-Update airflow dependency to 2.6.3.