dkist-processing-ops 1.0.0rc2__tar.gz → 1.0.0rc4__tar.gz

{dkist-processing-ops-1.0.0rc2 → dkist-processing-ops-1.0.0rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-ops
-Version: 1.0.0rc2
+Version: 1.0.0rc4
 Summary: Automated Processing smoke test and operations workflows
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD 3-Clause
@@ -11,7 +11,7 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.11
 Description-Content-Type: text/x-rst
 License-File: LICENSE.rst
-Requires-Dist: dkist-processing-core==3.0.1
+Requires-Dist: dkist-processing-common==6.1.0
 Requires-Dist: dkist-service-configuration==1.1.0
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"

{dkist-processing-ops-1.0.0rc2 → dkist-processing-ops-1.0.0rc4}/bitbucket-pipelines.yml

@@ -50,6 +50,7 @@ definitions:
           - export BUILD_VERSION="${BITBUCKET_TAG:1}"
           - export ARTIFACT_FOLDER="${BITBUCKET_REPO_SLUG}_${BUILD_VERSION}/"
           - python -c "from dkist_processing_core.build_utils import export_dags; import dkist_processing_ops.workflows as workflow_package; export_dags(workflow_package, '${ARTIFACT_FOLDER}')"
+          - python -c "from dkist_processing_ops.dags.scale import export_scale_dags; export_scale_dags('${ARTIFACT_FOLDER}')"
           - export SOURCE_PATH="workflow_${BUILD_VERSION}.gz"
           - tar --exclude="bitbucket-pipelines.yml" -cvzf ${SOURCE_PATH} ${ARTIFACT_FOLDER}
           - export TARGET_PATH="generic-packages/dkist-processing-ops/${BUILD_VERSION}/"

{dkist-processing-ops-1.0.0rc2 → dkist-processing-ops-1.0.0rc4}/dkist_processing_ops/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.0.0rc2'
+__version__ = version = '1.0.0rc4'
 __version_tuple__ = version_tuple = (1, 0, 0)

dkist-processing-ops-1.0.0rc4/dkist_processing_ops/dags/scale.py

@@ -0,0 +1,78 @@
+"""
+DAG to use up workers to support scaling
+"""
+from os import environ
+from pathlib import Path
+
+from dkist_processing_core.build_utils import export_dags
+
+
+def export_scale_dags(path: Path | str) -> list[Path]:
+    """Export all the ops dags"""
+    result = []
+    dag_prefix = "ops_scale"
+    scales = [16, 32]
+    queues = ["default", "high_memory"]
+    sleep_duration_seconds = 60
+    for queue in queues:
+        for scale in scales:
+            dag_name = f"{dag_prefix}_{queue}_{scale}"
+            dag_body = _scale_dag(
+                dag_name=dag_name,
+                sleep_duration_seconds=sleep_duration_seconds,
+                queue=queue,
+                concurrent_task_count=scale,
+            )
+            dag_path = _export_ops_dag(dag_name=dag_name, dag_body=dag_body, path=path)
+            result.append(dag_path)
+    return result
+
+
+def _export_ops_dag(dag_name: str, dag_body: str, path: Path | str | None = None) -> Path:
+    """Write a file representation of the scaling DAG."""
+    path = path or "dags/"
+    path = Path(path)
+    path.mkdir(exist_ok=True)
+    version = environ.get("BUILD_VERSION", "dev")
+    dag_name = f"{dag_name}_{version}"
+    workflow_py = path / f"{dag_name}.py"
+    with workflow_py.open(mode="w") as f:
+        f.write(dag_body)
+    return workflow_py
+
+
+def _scale_dag(
+    dag_name: str,
+    sleep_duration_seconds: int = 60,
+    queue: str | None = None,
+    concurrent_task_count: int = 16,
+) -> str:
+    queue = queue or "default"
+
+    imports = f"""# Scale {concurrent_task_count} DAG on queue {queue}
+from datetime import timedelta
+import pendulum
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+"""
+    dag = f"""with DAG(
+        dag_id="{dag_name}",
+        start_date=pendulum.today("UTC").add(days=-2),
+        schedule=None,
+        catchup=False,
+        tags=["ops", "scale"],
+    ) as d:"""
+    tasks = []
+    for idx in range(concurrent_task_count):
+        task = f"""    t{idx} = BashOperator(
+            task_id="t{idx}",
+            bash_command=f"sleep {sleep_duration_seconds}",
+            retries=0,
+            retry_delay=timedelta(seconds=60),
+            owner="DKIST Data Center",
+            queue="{queue}",
+        )"""
+        tasks.append(task)
+    parts = [imports, dag] + tasks
+    body = "\n".join(parts)
+    return body

dkist-processing-ops-1.0.0rc4/dkist_processing_ops/tasks/wait.py

@@ -0,0 +1,15 @@
+"""Task for parallelization testing which sleeps a configurable amount of time"""
+from time import sleep
+
+from dkist_processing_core import TaskBase
+
+
+__all__ = ["WaitTask"]
+
+
+SLEEP_TIME = 60
+
+
+class WaitTask(TaskBase):
+    def run(self) -> None:
+        sleep(SLEEP_TIME)

dkist-processing-ops-1.0.0rc4/dkist_processing_ops/workflows/smoke.py

@@ -0,0 +1,28 @@
+"""Workflows to test task submission and spin up"""
+from dkist_processing_common.tasks import TrialTeardown
+from dkist_processing_core import ResourceQueue
+from dkist_processing_core import Workflow
+
+from dkist_processing_ops.tasks import WaitTask
+
+
+smoke_default = Workflow(
+    input_data="ops",
+    output_data="common",
+    category="smoke",
+    detail="default",
+    workflow_package=__package__,
+)
+smoke_default.add_node(task=WaitTask, upstreams=None, resource_queue=ResourceQueue.DEFAULT)
+smoke_default.add_node(task=TrialTeardown, upstreams=WaitTask)
+
+
+smoke_high_mem = Workflow(
+    input_data="ops",
+    output_data="common",
+    category="smoke",
+    detail="high-mem",
+    workflow_package=__package__,
+)
+smoke_high_mem.add_node(task=WaitTask, upstreams=None, resource_queue=ResourceQueue.HIGH_MEMORY)
+smoke_high_mem.add_node(task=TrialTeardown, upstreams=WaitTask)

{dkist-processing-ops-1.0.0rc2 → dkist-processing-ops-1.0.0rc4}/dkist_processing_ops.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-processing-ops
-Version: 1.0.0rc2
+Version: 1.0.0rc4
 Summary: Automated Processing smoke test and operations workflows
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD 3-Clause
@@ -11,7 +11,7 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.11
 Description-Content-Type: text/x-rst
 License-File: LICENSE.rst
-Requires-Dist: dkist-processing-core==3.0.1
+Requires-Dist: dkist-processing-common==6.1.0
 Requires-Dist: dkist-service-configuration==1.1.0
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"

{dkist-processing-ops-1.0.0rc2 → dkist-processing-ops-1.0.0rc4}/dkist_processing_ops.egg-info/SOURCES.txt

@@ -13,9 +13,10 @@ dkist_processing_ops.egg-info/dependency_links.txt
 dkist_processing_ops.egg-info/not-zip-safe
 dkist_processing_ops.egg-info/requires.txt
 dkist_processing_ops.egg-info/top_level.txt
+dkist_processing_ops/dags/scale.py
 dkist_processing_ops/tasks/__init__.py
 dkist_processing_ops/tasks/wait.py
 dkist_processing_ops/tests/__init__.py
 dkist_processing_ops/tests/test_workflows.py
 dkist_processing_ops/workflows/__init__.py
-dkist_processing_ops/workflows/scale.py
+dkist_processing_ops/workflows/smoke.py

{dkist-processing-ops-1.0.0rc2 → dkist-processing-ops-1.0.0rc4}/dkist_processing_ops.egg-info/requires.txt

@@ -1,4 +1,4 @@
-dkist-processing-core==3.0.1
+dkist-processing-common==6.1.0
 dkist-service-configuration==1.1.0
 
 [test]

{dkist-processing-ops-1.0.0rc2 → dkist-processing-ops-1.0.0rc4}/pyproject.toml

@@ -23,7 +23,7 @@ authors = [
 ]
 
 dependencies = [
-    "dkist-processing-core==3.0.1",
+    "dkist-processing-common==6.1.0",
     "dkist-service-configuration==1.1.0",
 ]
 dynamic = ["version"]

dkist-processing-ops-1.0.0rc2/dkist_processing_ops/tasks/wait.py

@@ -1,151 +0,0 @@
-"""Task for parallelization testing which sleeps a configurable amount of time"""
-from time import sleep
-
-from dkist_processing_core import TaskBase
-
-
-__all__ = [f"WaitTask{i}" for i in range(32)]
-
-
-SLEEP_TIME = 300
-
-
-class WaitTaskBase(TaskBase):
-    def __init__(
-        self,
-        recipe_run_id: int = 0,
-        workflow_name: str = "ops",
-        workflow_version: str = "ops_ver",
-    ):
-        super().__init__(recipe_run_id, workflow_name, workflow_version)
-
-    def run(self) -> None:
-        sleep(SLEEP_TIME)
-
-
-class WaitTask0(WaitTaskBase):
-    pass
-
-
-class WaitTask1(WaitTaskBase):
-    pass
-
-
-class WaitTask2(WaitTaskBase):
-    pass
-
-
-class WaitTask3(WaitTaskBase):
-    pass
-
-
-class WaitTask4(WaitTaskBase):
-    pass
-
-
-class WaitTask5(WaitTaskBase):
-    pass
-
-
-class WaitTask6(WaitTaskBase):
-    pass
-
-
-class WaitTask7(WaitTaskBase):
-    pass
-
-
-class WaitTask8(WaitTaskBase):
-    pass
-
-
-class WaitTask9(WaitTaskBase):
-    pass
-
-
-class WaitTask10(WaitTaskBase):
-    pass
-
-
-class WaitTask11(WaitTaskBase):
-    pass
-
-
-class WaitTask12(WaitTaskBase):
-    pass
-
-
-class WaitTask13(WaitTaskBase):
-    pass
-
-
-class WaitTask14(WaitTaskBase):
-    pass
-
-
-class WaitTask15(WaitTaskBase):
-    pass
-
-
-class WaitTask16(WaitTaskBase):
-    pass
-
-
-class WaitTask17(WaitTaskBase):
-    pass
-
-
-class WaitTask18(WaitTaskBase):
-    pass
-
-
-class WaitTask19(WaitTaskBase):
-    pass
-
-
-class WaitTask20(WaitTaskBase):
-    pass
-
-
-class WaitTask21(WaitTaskBase):
-    pass
-
-
-class WaitTask22(WaitTaskBase):
-    pass
-
-
-class WaitTask23(WaitTaskBase):
-    pass
-
-
-class WaitTask24(WaitTaskBase):
-    pass
-
-
-class WaitTask25(WaitTaskBase):
-    pass
-
-
-class WaitTask26(WaitTaskBase):
-    pass
-
-
-class WaitTask27(WaitTaskBase):
-    pass
-
-
-class WaitTask28(WaitTaskBase):
-    pass
-
-
-class WaitTask29(WaitTaskBase):
-    pass
-
-
-class WaitTask30(WaitTaskBase):
-    pass
-
-
-class WaitTask31(WaitTaskBase):
-    pass

dkist-processing-ops-1.0.0rc2/dkist_processing_ops/workflows/scale.py

@@ -1,206 +0,0 @@
-"""Workflows to test parallel scaling."""
-from dkist_processing_core import ResourceQueue
-from dkist_processing_core import Workflow
-
-from dkist_processing_ops.tasks import WaitTask0
-from dkist_processing_ops.tasks import WaitTask1
-from dkist_processing_ops.tasks import WaitTask10
-from dkist_processing_ops.tasks import WaitTask11
-from dkist_processing_ops.tasks import WaitTask12
-from dkist_processing_ops.tasks import WaitTask13
-from dkist_processing_ops.tasks import WaitTask14
-from dkist_processing_ops.tasks import WaitTask15
-from dkist_processing_ops.tasks import WaitTask16
-from dkist_processing_ops.tasks import WaitTask17
-from dkist_processing_ops.tasks import WaitTask18
-from dkist_processing_ops.tasks import WaitTask19
-from dkist_processing_ops.tasks import WaitTask2
-from dkist_processing_ops.tasks import WaitTask20
-from dkist_processing_ops.tasks import WaitTask21
-from dkist_processing_ops.tasks import WaitTask22
-from dkist_processing_ops.tasks import WaitTask23
-from dkist_processing_ops.tasks import WaitTask24
-from dkist_processing_ops.tasks import WaitTask25
-from dkist_processing_ops.tasks import WaitTask26
-from dkist_processing_ops.tasks import WaitTask27
-from dkist_processing_ops.tasks import WaitTask28
-from dkist_processing_ops.tasks import WaitTask29
-from dkist_processing_ops.tasks import WaitTask3
-from dkist_processing_ops.tasks import WaitTask30
-from dkist_processing_ops.tasks import WaitTask31
-from dkist_processing_ops.tasks import WaitTask4
-from dkist_processing_ops.tasks import WaitTask5
-from dkist_processing_ops.tasks import WaitTask6
-from dkist_processing_ops.tasks import WaitTask7
-from dkist_processing_ops.tasks import WaitTask8
-from dkist_processing_ops.tasks import WaitTask9
-
-
-ALL_WAIT_TASKS = [
-    WaitTask0,
-    WaitTask1,
-    WaitTask2,
-    WaitTask3,
-    WaitTask4,
-    WaitTask5,
-    WaitTask6,
-    WaitTask7,
-    WaitTask8,
-    WaitTask9,
-    WaitTask10,
-    WaitTask11,
-    WaitTask12,
-    WaitTask13,
-    WaitTask14,
-    WaitTask15,
-    WaitTask16,
-    WaitTask17,
-    WaitTask18,
-    WaitTask19,
-    WaitTask20,
-    WaitTask21,
-    WaitTask22,
-    WaitTask23,
-    WaitTask24,
-    WaitTask25,
-    WaitTask26,
-    WaitTask27,
-    WaitTask28,
-    WaitTask29,
-    WaitTask30,
-    WaitTask31,
-]
-
-
-#
-# Default Resource Queue
-#
-
-
-def add_parallel_nodes(count: int, workflow: Workflow, resource_queue: ResourceQueue):
-    """Add the 'count' number of nodes to run in parallel to a workflow"""
-    for task in ALL_WAIT_TASKS[:count]:
-        workflow.add_node(task=task, upstreams=None, resource_queue=resource_queue)
-
-
-single_default = Workflow(
-    input_data="ops",
-    output_data="scale",
-    category="default",
-    detail="1",
-    workflow_package=__package__,
-)
-add_parallel_nodes(count=1, workflow=single_default, resource_queue=ResourceQueue.DEFAULT)
-
-
-two_default = Workflow(
-    input_data="ops",
-    output_data="scale",
-    category="default",
-    detail="2",
-    workflow_package=__package__,
-)
-add_parallel_nodes(count=2, workflow=two_default, resource_queue=ResourceQueue.DEFAULT)
-
-
-four_default = Workflow(
-    input_data="ops",
-    output_data="scale",
-    category="default",
-    detail="4",
-    workflow_package=__package__,
-)
-add_parallel_nodes(count=4, workflow=four_default, resource_queue=ResourceQueue.DEFAULT)
-
-
-eight_default = Workflow(
-    input_data="ops",
-    output_data="scale",
-    category="default",
-    detail="8",
-    workflow_package=__package__,
-)
-add_parallel_nodes(count=8, workflow=eight_default, resource_queue=ResourceQueue.DEFAULT)
-
-
-sixteen_default = Workflow(
-    input_data="ops",
-    output_data="scale",
-    category="default",
-    detail="16",
-    workflow_package=__package__,
-)
-add_parallel_nodes(count=16, workflow=sixteen_default, resource_queue=ResourceQueue.DEFAULT)
-
-
-thirty_two_default = Workflow(
-    input_data="ops",
-    output_data="scale",
-    category="default",
-    detail="32",
-    workflow_package=__package__,
-)
-add_parallel_nodes(count=32, workflow=thirty_two_default, resource_queue=ResourceQueue.DEFAULT)
-
-#
-# High Mem Resource Queue
-#
-
-single_high_mem = Workflow(
-    input_data="ops",
-    output_data="scale",
-    category="high_mem",
-    detail="1",
-    workflow_package=__package__,
-)
-add_parallel_nodes(count=1, workflow=single_high_mem, resource_queue=ResourceQueue.HIGH_MEMORY)
-
-
-two_high_mem = Workflow(
-    input_data="ops",
-    output_data="scale",
-    category="high_mem",
-    detail="2",
-    workflow_package=__package__,
-)
-add_parallel_nodes(count=2, workflow=two_high_mem, resource_queue=ResourceQueue.HIGH_MEMORY)
-
-
-four_high_mem = Workflow(
-    input_data="ops",
-    output_data="scale",
-    category="high_mem",
-    detail="4",
-    workflow_package=__package__,
-)
-add_parallel_nodes(count=4, workflow=four_high_mem, resource_queue=ResourceQueue.HIGH_MEMORY)
-
-
-eight_high_mem = Workflow(
-    input_data="ops",
-    output_data="scale",
-    category="high_mem",
-    detail="8",
-    workflow_package=__package__,
-)
-add_parallel_nodes(count=8, workflow=eight_high_mem, resource_queue=ResourceQueue.HIGH_MEMORY)
-
-
-sixteen_high_mem = Workflow(
-    input_data="ops",
-    output_data="scale",
-    category="high_mem",
-    detail="16",
-    workflow_package=__package__,
-)
-add_parallel_nodes(count=16, workflow=sixteen_high_mem, resource_queue=ResourceQueue.HIGH_MEMORY)
-
-
-thirty_two_high_mem = Workflow(
-    input_data="ops",
-    output_data="scale",
-    category="high_mem",
-    detail="32",
-    workflow_package=__package__,
-)
-add_parallel_nodes(count=32, workflow=thirty_two_high_mem, resource_queue=ResourceQueue.HIGH_MEMORY)