dkist-processing-test 1.44.1__tar.gz → 1.44.2rc1__tar.gz

{dkist_processing_test-1.44.1 → dkist_processing_test-1.44.2rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-test
-Version: 1.44.1
+Version: 1.44.2rc1
 Summary: Example instrument code used by the DKIST science data processing pipelines to test processing infrastructure
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -30,6 +30,7 @@ Requires-Dist: dkist-processing-test[inventory]; extra == "test"
 Requires-Dist: dkist-processing-test[asdf]; extra == "test"
 Requires-Dist: dkist-processing-test[quality]; extra == "test"
 Requires-Dist: dkist-processing-test[gpu]; extra == "test"
+Requires-Dist: dkist-processing-test[dask]; extra == "test"
 Provides-Extra: inventory
 Requires-Dist: dkist-processing-common[inventory]; extra == "inventory"
 Requires-Dist: dkist-inventory==1.11.9; extra == "inventory"
@@ -40,6 +41,8 @@ Provides-Extra: quality
 Requires-Dist: dkist-quality==2.0.1; extra == "quality"
 Provides-Extra: gpu
 Requires-Dist: cupy-cuda12x[ctk]==14.1.1; extra == "gpu"
+Provides-Extra: dask
+Requires-Dist: dask; extra == "dask"
 Provides-Extra: frozen
 Requires-Dist: Deprecated==1.3.1; extra == "frozen"
 Requires-Dist: ImageIO==2.37.4; extra == "frozen"
@@ -93,8 +96,8 @@ Requires-Dist: babel==2.18.0; extra == "frozen"
 Requires-Dist: beautifulsoup4==4.15.0; extra == "frozen"
 Requires-Dist: billiard==4.2.4; extra == "frozen"
 Requires-Dist: bleach==6.4.0; extra == "frozen"
-Requires-Dist: boto3==1.43.71; extra == "frozen"
-Requires-Dist: botocore==1.43.71; extra == "frozen"
+Requires-Dist: boto3==1.43.72; extra == "frozen"
+Requires-Dist: botocore==1.43.72; extra == "frozen"
 Requires-Dist: cachetools==7.1.7; extra == "frozen"
 Requires-Dist: cadwyn==7.0.0; extra == "frozen"
 Requires-Dist: celery==5.6.3; extra == "frozen"
@@ -120,7 +123,7 @@ Requires-Dist: dkist-header-validator==5.4.0; extra == "frozen"
 Requires-Dist: dkist-processing-common==14.3.6; extra == "frozen"
 Requires-Dist: dkist-processing-core==7.5.0; extra == "frozen"
 Requires-Dist: dkist-processing-pac==3.1.2; extra == "frozen"
-Requires-Dist: dkist-processing-test==1.44.1; extra == "frozen"
+Requires-Dist: dkist-processing-test==1.44.2rc1; extra == "frozen"
 Requires-Dist: dkist-service-configuration==4.3.0; extra == "frozen"
 Requires-Dist: dkist-spectral-lines==3.0.1; extra == "frozen"
 Requires-Dist: dkist_fits_specifications==4.28.1; extra == "frozen"

{dkist_processing_test-1.44.1 → dkist_processing_test-1.44.2rc1}/bitbucket-pipelines.yml

@@ -58,7 +58,7 @@ definitions:
         name: Check frozen install with extras
         script:
           - pip install -U pip
-          - pip install .[asdf,quality,inventory,frozen,gpu]
+          - pip install .[asdf,quality,inventory,dask,frozen,gpu]
 
     - step: &push_workflow
         caches:

{dkist_processing_test-1.44.1 → dkist_processing_test-1.44.2rc1}/dkist_processing_test/tasks/exercise_gpu.py

@@ -6,6 +6,15 @@ from dkist_service_configuration.logging import logger
 
 __all__ = ["ExerciseGpu"]
 
+# Capture condition of dkist-processing-test[gpu] install.
+GPU_EXTRA_INSTALLED = False
+try:
+    import cupy as cp
+
+    GPU_EXTRA_INSTALLED = True
+except ModuleNotFoundError:
+    pass
+
 ELEMENT_COUNT = 1_000_000
 SCALE_FACTOR = np.float32(3.0)
 OFFSET = np.float32(7.0)
@@ -14,11 +23,17 @@ OFFSET = np.float32(7.0)
 class ExerciseGpu(TaskBase):
     """Execute CuPy operations on a GPU, then verify their result on the host."""
 
+    def pre_run(self) -> None:
+        """Require the GPU workflow extra be installed."""
+        if not GPU_EXTRA_INSTALLED:
+            raise ModuleNotFoundError(
+                f"{self.__class__.__name__} Task requires the cupy package "
+                f"(e.g. via a 'gpu' pip_extra on dkist_processing_core.Workflow().add_node()) "
+                f"but the required dependency was not found."
+            )
+
     def run(self) -> None:
         with self.telemetry_span("GPU Setup"):
-            # Import lazily so non-GPU tasks can be discovered without initializing CUDA.
-            import cupy as cp
-
             device_count = cp.cuda.runtime.getDeviceCount()
             if device_count < 1:
                 raise RuntimeError("CuPy did not find an available GPU.")

dkist_processing_test-1.44.2rc1/dkist_processing_test/tasks/exercise_multiprocessing.py

@@ -0,0 +1,121 @@
+"""Exercise CPU-level parallelism in the workflow execution environment."""
+
+import os
+from concurrent.futures import ProcessPoolExecutor
+
+from dkist_processing_core import TaskBase
+from dkist_service_configuration.logging import logger
+from pydantic import BaseModel
+
+from dkist_processing_test.config import dkist_processing_test_configurations
+
+__all__ = ["CpuBoundWorkloadResult", "ExerciseDask", "ExerciseMultiprocessing"]
+
+# Capture condition of dkist-processing-test[dask] install.
+DASK_EXTRA_INSTALLED = False
+try:
+    from dask import compute
+    from dask import delayed
+
+    DASK_EXTRA_INSTALLED = True
+except ModuleNotFoundError:
+    pass
+
+# Keep computation short while giving each worker process meaningful CPU work.
+WORKLOAD_SIZE = 1_000_000
+
+
+class CpuBoundWorkloadResult(BaseModel):
+    """Result from a CPU-bound workload executed in a worker process."""
+
+    workload_id: int
+    process_id: int
+    result: int
+
+
+def _cpu_bound_workload(workload_id: int) -> CpuBoundWorkloadResult:
+    """Run a deterministic CPU-bound workload and report the worker process."""
+    result = sum((value + workload_id) ** 2 for value in range(WORKLOAD_SIZE))
+    return CpuBoundWorkloadResult(
+        workload_id=workload_id,
+        process_id=os.getpid(),
+        result=result,
+    )
+
+
+def _expected_workload_result(workload_id: int) -> int:
+    """Calculate the closed-form result of ``_cpu_bound_workload``."""
+    value_sum = WORKLOAD_SIZE * (WORKLOAD_SIZE - 1) // 2
+    squared_value_sum = WORKLOAD_SIZE * (WORKLOAD_SIZE - 1) * (2 * WORKLOAD_SIZE - 1) // 6
+    return squared_value_sum + 2 * workload_id * value_sum + WORKLOAD_SIZE * workload_id**2
+
+
+class ExerciseMultiprocessing(TaskBase):
+    """Run one CPU-bound workload per configured CPU in separate worker processes."""
+
+    def run(self) -> None:
+        worker_count = dkist_processing_test_configurations.cpu_count
+
+        if worker_count <= 1:
+            raise RuntimeError(f"Insufficient CPU count for multiprocessing test. {worker_count=}")
+
+        with ProcessPoolExecutor(max_workers=worker_count) as executor:
+            workload_results = list(executor.map(_cpu_bound_workload, range(worker_count)))
+
+        worker_process_ids = {workload_result.process_id for workload_result in workload_results}
+        if len(worker_process_ids) != worker_count:
+            raise RuntimeError(
+                f"Expected {worker_count} worker processes; found {worker_process_ids}."
+            )
+
+        self.workload_results = workload_results
+        self.worker_process_ids = worker_process_ids
+        logger.info(
+            f"ExerciseMultiprocessing completed {worker_count} CPU-bound workloads "
+            f"using worker processes {sorted(worker_process_ids)}."
+        )
+
+
+class ExerciseDask(TaskBase):
+    """Schedule one CPU-bound workload per configured CPU using Dask processes."""
+
+    def pre_run(self) -> None:
+        """Require the Dask workflow extra be installed."""
+        if not DASK_EXTRA_INSTALLED:
+            raise ModuleNotFoundError(
+                f"{self.__class__.__name__} Task requires the dask package "
+                f"(e.g. via a 'dask' pip_extra on dkist_processing_core.Workflow().add_node()) "
+                f"but the required dependency was not found."
+            )
+
+    def run(self) -> None:
+        worker_count = dkist_processing_test_configurations.cpu_count
+        assert worker_count > 1, f"Insufficient CPU count for Dask test. {worker_count=}"
+
+        delayed_workloads = [
+            delayed(_cpu_bound_workload)(workload_id) for workload_id in range(worker_count)
+        ]
+        workload_results = list(
+            compute(
+                *delayed_workloads,
+                scheduler="processes",
+                num_workers=worker_count,
+            )
+        )
+
+        assert len(workload_results) == worker_count
+        assert {result.workload_id for result in workload_results} == set(range(worker_count))
+        assert all(
+            result.result == _expected_workload_result(result.workload_id)
+            for result in workload_results
+        )
+
+        worker_process_ids = {workload_result.process_id for workload_result in workload_results}
+        assert os.getpid() not in worker_process_ids
+
+        self.workload_results = workload_results
+        self.worker_process_ids = worker_process_ids
+        logger.info(
+            f"ExerciseDask completed {worker_count} CPU-bound workloads "
+            f"using worker processes {sorted(worker_process_ids)}."
+        )

dkist_processing_test-1.44.2rc1/dkist_processing_test/workflows/exercise_dask.py

@@ -0,0 +1,20 @@
+"""Workflow that exercises CPU-level multiprocessing with Dask."""
+
+from dkist_processing_core import ResourceQueue
+from dkist_processing_core import Workflow
+
+from dkist_processing_test.tasks import ExerciseDask
+
+exercise_dask = Workflow(
+    input_data="input",
+    output_data="output",
+    category="test",
+    detail="exercise_dask",
+    workflow_package=__package__,
+)
+exercise_dask.add_node(
+    task=ExerciseDask,
+    resource_queue=ResourceQueue.MULTI_CORE,
+    upstreams=None,
+    pip_extras=["dask"],
+)

{dkist_processing_test-1.44.1 → dkist_processing_test-1.44.2rc1}/dkist_processing_test.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-test
-Version: 1.44.1
+Version: 1.44.2rc1
 Summary: Example instrument code used by the DKIST science data processing pipelines to test processing infrastructure
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -30,6 +30,7 @@ Requires-Dist: dkist-processing-test[inventory]; extra == "test"
 Requires-Dist: dkist-processing-test[asdf]; extra == "test"
 Requires-Dist: dkist-processing-test[quality]; extra == "test"
 Requires-Dist: dkist-processing-test[gpu]; extra == "test"
+Requires-Dist: dkist-processing-test[dask]; extra == "test"
 Provides-Extra: inventory
 Requires-Dist: dkist-processing-common[inventory]; extra == "inventory"
 Requires-Dist: dkist-inventory==1.11.9; extra == "inventory"
@@ -40,6 +41,8 @@ Provides-Extra: quality
 Requires-Dist: dkist-quality==2.0.1; extra == "quality"
 Provides-Extra: gpu
 Requires-Dist: cupy-cuda12x[ctk]==14.1.1; extra == "gpu"
+Provides-Extra: dask
+Requires-Dist: dask; extra == "dask"
 Provides-Extra: frozen
 Requires-Dist: Deprecated==1.3.1; extra == "frozen"
 Requires-Dist: ImageIO==2.37.4; extra == "frozen"
@@ -93,8 +96,8 @@ Requires-Dist: babel==2.18.0; extra == "frozen"
 Requires-Dist: beautifulsoup4==4.15.0; extra == "frozen"
 Requires-Dist: billiard==4.2.4; extra == "frozen"
 Requires-Dist: bleach==6.4.0; extra == "frozen"
-Requires-Dist: boto3==1.43.71; extra == "frozen"
-Requires-Dist: botocore==1.43.71; extra == "frozen"
+Requires-Dist: boto3==1.43.72; extra == "frozen"
+Requires-Dist: botocore==1.43.72; extra == "frozen"
 Requires-Dist: cachetools==7.1.7; extra == "frozen"
 Requires-Dist: cadwyn==7.0.0; extra == "frozen"
 Requires-Dist: celery==5.6.3; extra == "frozen"
@@ -120,7 +123,7 @@ Requires-Dist: dkist-header-validator==5.4.0; extra == "frozen"
 Requires-Dist: dkist-processing-common==14.3.6; extra == "frozen"
 Requires-Dist: dkist-processing-core==7.5.0; extra == "frozen"
 Requires-Dist: dkist-processing-pac==3.1.2; extra == "frozen"
-Requires-Dist: dkist-processing-test==1.44.1; extra == "frozen"
+Requires-Dist: dkist-processing-test==1.44.2rc1; extra == "frozen"
 Requires-Dist: dkist-service-configuration==4.3.0; extra == "frozen"
 Requires-Dist: dkist-spectral-lines==3.0.1; extra == "frozen"
 Requires-Dist: dkist_fits_specifications==4.28.1; extra == "frozen"

{dkist_processing_test-1.44.1 → dkist_processing_test-1.44.2rc1}/dkist_processing_test.egg-info/SOURCES.txt

@@ -41,6 +41,7 @@ dkist_processing_test/tests/test_workflows.py
 dkist_processing_test/workflows/__init__.py
 dkist_processing_test/workflows/common_tasks.py
 dkist_processing_test/workflows/end_to_end.py
+dkist_processing_test/workflows/exercise_dask.py
 dkist_processing_test/workflows/exercise_gpu.py
 dkist_processing_test/workflows/exercise_multiprocessing.py
 dkist_processing_test/workflows/exercise_numba.py

{dkist_processing_test-1.44.1 → dkist_processing_test-1.44.2rc1}/dkist_processing_test.egg-info/requires.txt

@@ -11,6 +11,9 @@ dkist-spectral-lines==3.0.1
 dkist-processing-common[asdf]
 dkist-inventory[asdf]==1.11.9
 
+[dask]
+dask
+
 [frozen]
 Deprecated==1.3.1
 ImageIO==2.37.4
@@ -64,8 +67,8 @@ babel==2.18.0
 beautifulsoup4==4.15.0
 billiard==4.2.4
 bleach==6.4.0
-boto3==1.43.71
-botocore==1.43.71
+boto3==1.43.72
+botocore==1.43.72
 cachetools==7.1.7
 cadwyn==7.0.0
 celery==5.6.3
@@ -91,7 +94,7 @@ dkist-header-validator==5.4.0
 dkist-processing-common==14.3.6
 dkist-processing-core==7.5.0
 dkist-processing-pac==3.1.2
-dkist-processing-test==1.44.1
+dkist-processing-test==1.44.2rc1
 dkist-service-configuration==4.3.0
 dkist-spectral-lines==3.0.1
 dkist_fits_specifications==4.28.1
@@ -321,3 +324,4 @@ dkist-processing-test[inventory]
 dkist-processing-test[asdf]
 dkist-processing-test[quality]
 dkist-processing-test[gpu]
+dkist-processing-test[dask]

{dkist_processing_test-1.44.1 → dkist_processing_test-1.44.2rc1}/dkist_processing_test.egg-info/scm_file_list.json

@@ -35,6 +35,7 @@
     "dkist_processing_test/workflows/__init__.py",
     "dkist_processing_test/workflows/common_tasks.py",
     "dkist_processing_test/workflows/end_to_end.py",
+    "dkist_processing_test/workflows/exercise_dask.py",
     "dkist_processing_test/workflows/exercise_gpu.py",
     "dkist_processing_test/workflows/exercise_multiprocessing.py",
     "dkist_processing_test/workflows/exercise_numba.py",

{dkist_processing_test-1.44.1 → dkist_processing_test-1.44.2rc1}/dkist_processing_test.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "1.44.1",
+  "tag": "1.44.2rc1",
   "distance": 0,
-  "node": "g22486fbbd2b5ac4cb732f4062a4805bb02326f89",
+  "node": "ge56d8dd87eabef2bf30d2691b9be152394f93a69",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-08-14"

{dkist_processing_test-1.44.1 → dkist_processing_test-1.44.2rc1}/pyproject.toml

@@ -9,8 +9,8 @@ exclude = ["build*"]
 
 [tool.dkist-dev-tools]
 # Most recently frozen version by dkist-dev-tools
-version = "1.44.1"
-date = 2026-08-14T09:30:02.796098
+version = "1.44.2rc1"
+date = 2026-08-14T15:27:21.539269
 
 [project]
 dynamic = ["version"]
@@ -52,6 +52,7 @@ test = [
   "dkist-processing-test[asdf]",
   "dkist-processing-test[quality]",
   "dkist-processing-test[gpu]",
+  "dkist-processing-test[dask]",
 ]
 inventory = [
   "dkist-processing-common[inventory]",
@@ -67,6 +68,9 @@ quality = [
 gpu = [
   "cupy-cuda12x[ctk] == 14.1.1",
 ]
+dask = [
+  "dask",
+]
 frozen = [
     "Deprecated == 1.3.1",
     "ImageIO == 2.37.4",
@@ -120,8 +124,8 @@ frozen = [
     "beautifulsoup4 == 4.15.0",
     "billiard == 4.2.4",
     "bleach == 6.4.0",
-    "boto3 == 1.43.71",
-    "botocore == 1.43.71",
+    "boto3 == 1.43.72",
+    "botocore == 1.43.72",
     "cachetools == 7.1.7",
     "cadwyn == 7.0.0",
     "celery == 5.6.3",
@@ -147,7 +151,7 @@ frozen = [
     "dkist-processing-common == 14.3.6",
     "dkist-processing-core == 7.5.0",
     "dkist-processing-pac == 3.1.2",
-    "dkist-processing-test == 1.44.1",
+    "dkist-processing-test == 1.44.2rc1",
     "dkist-service-configuration == 4.3.0",
     "dkist-spectral-lines == 3.0.1",
     "dkist_fits_specifications == 4.28.1",

dkist_processing_test-1.44.1/dkist_processing_test/tasks/exercise_multiprocessing.py

@@ -1,59 +0,0 @@
-"""Exercise CPU-level multiprocessing in the workflow execution environment."""
-
-import os
-from concurrent.futures import ProcessPoolExecutor
-
-from dkist_processing_core import TaskBase
-from dkist_service_configuration.logging import logger
-from pydantic import BaseModel
-
-from dkist_processing_test.config import dkist_processing_test_configurations
-
-__all__ = ["CpuBoundWorkloadResult", "ExerciseMultiprocessing"]
-
-# Keep computation short while giving each worker process meaningful CPU work.
-WORKLOAD_SIZE = 1_000_000
-
-
-class CpuBoundWorkloadResult(BaseModel):
-    """Result from a CPU-bound workload executed in a worker process."""
-
-    workload_id: int
-    process_id: int
-    result: int
-
-
-def _cpu_bound_workload(workload_id: int) -> CpuBoundWorkloadResult:
-    """Run a deterministic CPU-bound workload and report the worker process."""
-    result = sum((value + workload_id) ** 2 for value in range(WORKLOAD_SIZE))
-    return CpuBoundWorkloadResult(
-        workload_id=workload_id,
-        process_id=os.getpid(),
-        result=result,
-    )
-
-
-class ExerciseMultiprocessing(TaskBase):
-    """Run one CPU-bound workload per configured CPU in separate worker processes."""
-
-    def run(self) -> None:
-        worker_count = dkist_processing_test_configurations.cpu_count
-
-        if worker_count <= 1:
-            raise RuntimeError(f"Insufficient CPU count for multiprocessing test. {worker_count=}")
-
-        with ProcessPoolExecutor(max_workers=worker_count) as executor:
-            workload_results = list(executor.map(_cpu_bound_workload, range(worker_count)))
-
-        worker_process_ids = {workload_result.process_id for workload_result in workload_results}
-        if len(worker_process_ids) != worker_count:
-            raise RuntimeError(
-                f"Expected {worker_count} worker processes; found {worker_process_ids}."
-            )
-
-        self.workload_results = workload_results
-        self.worker_process_ids = worker_process_ids
-        logger.info(
-            f"ExerciseMultiprocessing completed {worker_count} CPU-bound workloads "
-            f"using worker processes {sorted(worker_process_ids)}."
-        )