dkist-processing-core 2.1.2__tar.gz → 2.1.3rc1__tar.gz

{dkist-processing-core-2.1.2 → dkist-processing-core-2.1.3rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-core
-Version: 2.1.2
+Version: 2.1.3rc1
 Summary: Abstraction layer that is used by the DKIST Science Data Processing pipelines to process DKIST data using Apache Airflow.
 Home-page: https://bitbucket.org/dkistdc/dkist-processing-core/src/main/
 Author: NSO / AURA

dkist-processing-core-2.1.3rc1/changelog/23.misc.rst

@@ -0,0 +1 @@
+Support specifying pip extras for individual nodes in a workflow.

{dkist-processing-core-2.1.2 → dkist-processing-core-2.1.3rc1}/dkist_processing_core/_node.py

@@ -23,6 +23,7 @@ class Node:
         task: task_type_hint,
         resource_queue: ResourceQueue,
         upstreams: upstreams_type_hint = None,
+        pip_extras: list[str] | None = None,
     ):
         """Node setup."""
         # Task type checking
@@ -43,6 +44,7 @@ class Node:
         self.workflow_package = workflow_package
         self.upstreams = upstreams
         self.resource_queue = resource_queue
+        self.pip_extras = pip_extras
 
     @property
     def operator(self) -> BashOperator:
@@ -117,12 +119,20 @@ rm -rf .task_venv
 echo Exit with code from main command: $exit_code
 exit $exit_code"""
 
+    @property
+    def _formatted_pip_extras(self) -> str:
+        if self.pip_extras:
+            extra_requirements = ",".join(self.pip_extras)
+            return f"'[{extra_requirements}]'"
+        return ""
+
     @property
     def _install_command(self) -> str:
         repo_name = self.workflow_package.split(".")[0].replace("_", "-")
         version = self.workflow_version
+        extras = self._formatted_pip_extras
         return f"""python -m pip install --upgrade pip
-python -m pip install {repo_name}=={version}"""
+python -m pip install {repo_name}{extras}=={version}"""
 
     @property
     def _run_command(self) -> str:

{dkist-processing-core-2.1.2 → dkist-processing-core-2.1.3rc1}/dkist_processing_core/tests/conftest.py

@@ -113,8 +113,18 @@ def queue_name(request):
     return ResourceQueue.HIGH_MEMORY
 
 
+@pytest.fixture(params=["default", "non_default"])
+def pip_extras(request):
+    """Extra pip requirements for Node initialization"""
+    if request.param == "default":
+        return None
+    return ["asdf"]
+
+
 @pytest.fixture(params=["0_upstream", "1_upstream", "2_upstream"])
-def node(workflow_tasks, request, queue_name) -> tuple[Node, task_type_hint, Any, str, str]:
+def node(
+    workflow_tasks, request, queue_name, pip_extras
+) -> tuple[Node, task_type_hint, Any, str, str]:
     """Node instance and its component parts."""
     version = "V6-123"
     name = f"{request.param}_{version}"
@@ -134,6 +144,7 @@ def node(workflow_tasks, request, queue_name) -> tuple[Node, task_type_hint, Any
             task=TaskA,
             upstreams=upstream[0],
             resource_queue=queue_name,
+            pip_extras=pip_extras,
         ),
         TaskA,
         upstream[1],

{dkist-processing-core-2.1.2 → dkist-processing-core-2.1.3rc1}/dkist_processing_core/tests/test_node.py

@@ -13,7 +13,7 @@ from dkist_processing_core.tests.task_example import Task
 
 
 @pytest.fixture()
-def single_node() -> Node:
+def single_node(pip_extras) -> Node:
     """A single Node instance."""
     return Node(
         workflow_name="test_node",
@@ -21,10 +21,11 @@ def single_node() -> Node:
         workflow_package=__package__,
         task=Task,
         resource_queue=ResourceQueue.DEFAULT,
+        pip_extras=pip_extras,
     )
 
 
-def test_nodes(node, fake_producer_factory, queue_name):
+def test_nodes(node, fake_producer_factory, queue_name, pip_extras):
     """
     Given: Workflow tasks to initialize a Node.
     When: Initializing the Node with valid task and upstreams.
@@ -44,6 +45,7 @@ def test_nodes(node, fake_producer_factory, queue_name):
     assert node.task == task
     assert node.workflow_version == version
     assert node.resource_queue == queue_name
+    assert node.pip_extras == pip_extras
 
 
 def test_node_bash_template_return_0(node):

{dkist-processing-core-2.1.2 → dkist-processing-core-2.1.3rc1}/dkist_processing_core/workflow.py

@@ -164,6 +164,7 @@ class Workflow:
         task: task_type_hint,
         upstreams: upstreams_type_hint = None,
         resource_queue: ResourceQueue | None = None,
+        pip_extras: list[str] | None = None,
     ) -> None:
         """Add a node and edges from that node to the workflow."""
         if resource_queue is None:
@@ -176,6 +177,7 @@ class Workflow:
             task=task,
             resource_queue=resource_queue,
             upstreams=upstreams,
+            pip_extras=pip_extras,
         )
         self.nodes.append(node)
         # confirm that the node can be properly added to a dag

{dkist-processing-core-2.1.2 → dkist-processing-core-2.1.3rc1}/dkist_processing_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-core
-Version: 2.1.2
+Version: 2.1.3rc1
 Summary: Abstraction layer that is used by the DKIST Science Data Processing pipelines to process DKIST data using Apache Airflow.
 Home-page: https://bitbucket.org/dkistdc/dkist-processing-core/src/main/
 Author: NSO / AURA

{dkist-processing-core-2.1.2 → dkist-processing-core-2.1.3rc1}/dkist_processing_core.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ pyproject.toml
 setup.cfg
 setup.py
 changelog/.gitempty
+changelog/23.misc.rst
 dkist_processing_core/__init__.py
 dkist_processing_core/_failure_callback.py
 dkist_processing_core/_node.py