dkist-processing-ops 1.0.0__py3-none-any.whl

dkist_processing_ops/_version.py

@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '1.0.0'
+__version_tuple__ = version_tuple = (1, 0, 0)

dkist_processing_ops/dags/scale.py

@@ -0,0 +1,78 @@
+"""
+DAG to use up workers to support scaling
+"""
+from os import environ
+from pathlib import Path
+
+from dkist_processing_core.build_utils import export_dags
+
+
+def export_scale_dags(path: Path | str) -> list[Path]:
+    """Export all the ops dags"""
+    result = []
+    dag_prefix = "ops_scale"
+    scales = [16, 32]
+    queues = ["default", "high_memory"]
+    sleep_duration_seconds = 60
+    for queue in queues:
+        for scale in scales:
+            dag_name = f"{dag_prefix}_{queue}_{scale}"
+            dag_body = _scale_dag(
+                dag_name=dag_name,
+                sleep_duration_seconds=sleep_duration_seconds,
+                queue=queue,
+                concurrent_task_count=scale,
+            )
+            dag_path = _export_ops_dag(dag_name=dag_name, dag_body=dag_body, path=path)
+            result.append(dag_path)
+    return result
+
+
+def _export_ops_dag(dag_name: str, dag_body: str, path: Path | str | None = None) -> Path:
+    """Write a file representation of the scaling DAG."""
+    path = path or "dags/"
+    path = Path(path)
+    path.mkdir(exist_ok=True)
+    version = environ.get("BUILD_VERSION", "dev")
+    dag_name = f"{dag_name}_{version}"
+    workflow_py = path / f"{dag_name}.py"
+    with workflow_py.open(mode="w") as f:
+        f.write(dag_body)
+    return workflow_py
+
+
+def _scale_dag(
+    dag_name: str,
+    sleep_duration_seconds: int = 60,
+    queue: str | None = None,
+    concurrent_task_count: int = 16,
+) -> str:
+    queue = queue or "default"
+
+    imports = f"""# Scale {concurrent_task_count} DAG on queue {queue}
+from datetime import timedelta
+import pendulum
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+"""
+    dag = f"""with DAG(
+        dag_id="{dag_name}",
+        start_date=pendulum.today("UTC").add(days=-2),
+        schedule=None,
+        catchup=False,
+        tags=["ops", "scale"],
+    ) as d:"""
+    tasks = []
+    for idx in range(concurrent_task_count):
+        task = f"""    t{idx} = BashOperator(
+            task_id="t{idx}",
+            bash_command=f"sleep {sleep_duration_seconds}",
+            retries=0,
+            retry_delay=timedelta(seconds=60),
+            owner="DKIST Data Center",
+            queue="{queue}",
+        )"""
+        tasks.append(task)
+    parts = [imports, dag] + tasks
+    body = "\n".join(parts)
+    return body

dkist_processing_ops/tasks/__init__.py

@@ -0,0 +1,2 @@
+"""Import of tasks is directly from package by convention"""
+from dkist_processing_ops.tasks.wait import *

dkist_processing_ops/tasks/wait.py

@@ -0,0 +1,15 @@
+"""Task for parallelization testing which sleeps a configurable amount of time"""
+from time import sleep
+
+from dkist_processing_core import TaskBase
+
+
+__all__ = ["WaitTask"]
+
+
+SLEEP_TIME = 60
+
+
+class WaitTask(TaskBase):
+    def run(self) -> None:
+        sleep(SLEEP_TIME)

dkist_processing_ops/tests/test_workflows.py

@@ -0,0 +1,9 @@
+"""Test integrity of workflows."""
+from dkist_processing_core.build_utils import validate_workflows
+
+from dkist_processing_ops import workflows
+
+
+def test_workflow_integrity():
+    """Validate workflow to ensure acyclic-ness and export compilation"""
+    validate_workflows(workflows)

dkist_processing_ops/workflows/smoke.py

@@ -0,0 +1,28 @@
+"""Workflows to test task submission and spin up"""
+from dkist_processing_common.tasks import TrialTeardown
+from dkist_processing_core import ResourceQueue
+from dkist_processing_core import Workflow
+
+from dkist_processing_ops.tasks import WaitTask
+
+
+smoke_default = Workflow(
+    input_data="ops",
+    output_data="common",
+    category="smoke",
+    detail="default",
+    workflow_package=__package__,
+)
+smoke_default.add_node(task=WaitTask, upstreams=None, resource_queue=ResourceQueue.DEFAULT)
+smoke_default.add_node(task=TrialTeardown, upstreams=WaitTask)
+
+
+smoke_high_mem = Workflow(
+    input_data="ops",
+    output_data="common",
+    category="smoke",
+    detail="high-mem",
+    workflow_package=__package__,
+)
+smoke_high_mem.add_node(task=WaitTask, upstreams=None, resource_queue=ResourceQueue.HIGH_MEMORY)
+smoke_high_mem.add_node(task=TrialTeardown, upstreams=WaitTask)

dkist_processing_ops-1.0.0.dist-info/LICENSE.rst

@@ -0,0 +1,8 @@
+License
+-------
+
+This project is Copyright (c) NSO / AURA and licensed under
+the terms of the BSD 3-Clause license. This package is based upon
+the `Openastronomy packaging guide <https://github.com/OpenAstronomy/packaging-guide>`_
+which is licensed under the BSD 3-clause licence. See the licenses folder for
+more information.

dkist_processing_ops-1.0.0.dist-info/METADATA

@@ -0,0 +1,41 @@
+Metadata-Version: 2.1
+Name: dkist-processing-ops
+Version: 1.0.0
+Summary: Automated Processing smoke test and operations workflows
+Author-email: NSO / AURA <dkistdc@nso.edu>
+License: BSD 3-Clause
+Project-URL: repository, https://bitbucket.org/dkistdc/dkist-processing-ops
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.11
+Description-Content-Type: text/x-rst
+License-File: LICENSE.rst
+Requires-Dist: dkist-processing-common ==6.1.0
+Requires-Dist: dkist-service-configuration ==1.1.0
+Provides-Extra: test
+Requires-Dist: pytest ; extra == 'test'
+Requires-Dist: pytest-cov ; extra == 'test'
+Requires-Dist: pytest-xdist ; extra == 'test'
+
+dkist-processing-ops
+--------------------
+|codecov|
+
+This repository works in concert with `dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_ and
+`dkist-processing-common <https://pypi.org/project/dkist-processing-common/>`_ to provide workflows for the
+operational management and smoke testing of the `Automated Processing <https://nso.atlassian.net/wiki/spaces/DPD/pages/3671451/04+-+Automated+Processing>`_ stack.
+
+
+Developer Setup
+~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    pip install -e .[test]
+    pip install pre-commit
+    pre-commit install
+
+
+.. |codecov| image:: https://codecov.io/bb/dkistdc/dkist-processing-ops/branch/main/graph/badge.svg
+   :target: https://codecov.io/bb/dkistdc/dkist-processing-ops

dkist_processing_ops-1.0.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+dkist_processing_ops/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dkist_processing_ops/_version.py,sha256=DGJ4pj32xs3_DRJhSzQwCiRNnAQrMgo09USYpyMZsKc,411
+dkist_processing_ops/dags/scale.py,sha256=GE3SzSEZT-UDnsL1U-DqLNxlW_VudNzXMLezSDDE_Tk,2341
+dkist_processing_ops/tasks/__init__.py,sha256=P81O9cg4dlBMqBTaWitdsAte68RsMtDlhV30JSZfXUY,107
+dkist_processing_ops/tasks/wait.py,sha256=uObka-nH1dKPcGBDsp3t2RCtTV2F1kksM0V-lRewFuY,273
+dkist_processing_ops/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dkist_processing_ops/tests/test_workflows.py,sha256=Ch_8BlGeQyPJU_9hB_GOncwW-SoZwpRUVKMOEz0RQZk,285
+dkist_processing_ops/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dkist_processing_ops/workflows/smoke.py,sha256=ofXu0_iYF6L3zQy-BOVvS5VdzKhmXs1gyugqMNkd-GM,878
+dkist_processing_ops-1.0.0.dist-info/LICENSE.rst,sha256=LJjTmkf2-q1phdZSySMpiyPxgLOy6zYHOr3R1Bb1__8,327
+dkist_processing_ops-1.0.0.dist-info/METADATA,sha256=nKfMDV6uFh1gswS0EGU8TykLQa44Y8JsEwgZcc4XQbg,1498
+dkist_processing_ops-1.0.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+dkist_processing_ops-1.0.0.dist-info/top_level.txt,sha256=o_SNho1HKt6wvCSUhm9qzX9FS2iopnqYuMos1CCD9cI,21
+dkist_processing_ops-1.0.0.dist-info/RECORD,,

dkist_processing_ops-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.43.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dkist_processing_ops-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+dkist_processing_ops