dkist-processing-ops 1.0.0__tar.gz

dkist-processing-ops-1.0.0/.gitignore

@@ -0,0 +1,161 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+dkist_quality/_version.py
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

dkist-processing-ops-1.0.0/.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.4.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+    -   id: debug-statements
+-   repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+    -   id: black
+        args: [ "-l 100" ]
+-   repo: https://github.com/asottile/reorder_python_imports
+    rev: v2.3.5
+    hooks:
+    -   id: reorder-python-imports

dkist-processing-ops-1.0.0/LICENSE.rst

@@ -0,0 +1,8 @@
+License
+-------
+
+This project is Copyright (c) NSO / AURA and licensed under
+the terms of the BSD 3-Clause license. This package is based upon
+the `Openastronomy packaging guide <https://github.com/OpenAstronomy/packaging-guide>`_
+which is licensed under the BSD 3-clause licence. See the licenses folder for
+more information.

dkist-processing-ops-1.0.0/MANIFEST.in

@@ -0,0 +1,19 @@
+# Include standard files
+include README.rst
+include CHANGES.rst
+include setup.cfg
+include LICENSE.rst
+include pyproject.toml
+
+# Exclude specific files
+# All files which are tracked by git and not explicitly excluded here are included by setuptools_scm
+
+# Prune folders
+prune build
+prune docs/_build
+prune docs/api
+global-exclude *.pyc *.o
+
+# This subpackage is only used in development checkouts
+# and should not be included in built tarballs
+prune dkist_quality/_dev

dkist-processing-ops-1.0.0/PKG-INFO

@@ -0,0 +1,41 @@
+Metadata-Version: 2.1
+Name: dkist-processing-ops
+Version: 1.0.0
+Summary: Automated Processing smoke test and operations workflows
+Author-email: NSO / AURA <dkistdc@nso.edu>
+License: BSD 3-Clause
+Project-URL: repository, https://bitbucket.org/dkistdc/dkist-processing-ops
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.11
+Description-Content-Type: text/x-rst
+License-File: LICENSE.rst
+Requires-Dist: dkist-processing-common==6.1.0
+Requires-Dist: dkist-service-configuration==1.1.0
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: pytest-xdist; extra == "test"
+
+dkist-processing-ops
+--------------------
+|codecov|
+
+This repository works in concert with `dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_ and
+`dkist-processing-common <https://pypi.org/project/dkist-processing-common/>`_ to provide workflows for the
+operational management and smoke testing of the `Automated Processing <https://nso.atlassian.net/wiki/spaces/DPD/pages/3671451/04+-+Automated+Processing>`_ stack.
+
+
+Developer Setup
+~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    pip install -e .[test]
+    pip install pre-commit
+    pre-commit install
+
+
+.. |codecov| image:: https://codecov.io/bb/dkistdc/dkist-processing-ops/branch/main/graph/badge.svg
+   :target: https://codecov.io/bb/dkistdc/dkist-processing-ops

dkist-processing-ops-1.0.0/README.rst

@@ -0,0 +1,21 @@
+dkist-processing-ops
+--------------------
+|codecov|
+
+This repository works in concert with `dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_ and
+`dkist-processing-common <https://pypi.org/project/dkist-processing-common/>`_ to provide workflows for the
+operational management and smoke testing of the `Automated Processing <https://nso.atlassian.net/wiki/spaces/DPD/pages/3671451/04+-+Automated+Processing>`_ stack.
+
+
+Developer Setup
+~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    pip install -e .[test]
+    pip install pre-commit
+    pre-commit install
+
+
+.. |codecov| image:: https://codecov.io/bb/dkistdc/dkist-processing-ops/branch/main/graph/badge.svg
+   :target: https://codecov.io/bb/dkistdc/dkist-processing-ops

dkist-processing-ops-1.0.0/bitbucket-pipelines.yml

@@ -0,0 +1,84 @@
+#Build Configuration for docker deployment to artifactory
+image: python:3.11
+
+definitions:
+  services:
+    redis:
+      image: redis
+  steps:
+    - step: &lint
+        caches:
+          - pip
+        name: Lint
+        script:
+          - pip install -U pip
+          - pip install pre-commit
+          - pre-commit install
+          - pre-commit run --all-files
+    - step: &scan
+        caches:
+          - pip
+        name: Scan
+        script:
+          - pip install -U pip
+          - pip install .
+          - pip freeze | grep -v @ > requirements.txt
+          - cat requirements.txt
+          - echo $SNYK_VERSION
+          - curl -L -o snyk https://github.com/snyk/snyk/releases/download/$SNYK_VERSION/snyk-linux
+          - chmod 755 snyk
+          - ./snyk -d auth $SNYK_TOKEN
+          - echo $SNYK_CLI_COMMAND
+          - $SNYK_CLI_COMMAND
+    - step: &test
+        caches:
+          - pip
+        name: Test
+        script:
+          - pip install -U pip
+          - pip install .[test]
+          - pytest -v -n auto -m "not development" --pyargs dkist_processing_ops --cov dkist_processing_ops --cov-report xml:coverage.xml
+        services:
+          - redis
+    - step: &push_workflow
+        caches:
+          - pip
+        name: Push Workflow
+        script:
+          - pip install -U pip
+          - pip install .
+          - export BUILD_VERSION="${BITBUCKET_TAG:1}"
+          - export ARTIFACT_FOLDER="${BITBUCKET_REPO_SLUG}_${BUILD_VERSION}/"
+          - python -c "from dkist_processing_core.build_utils import export_dags; import dkist_processing_ops.workflows as workflow_package; export_dags(workflow_package, '${ARTIFACT_FOLDER}')"
+          - python -c "from dkist_processing_ops.dags.scale import export_scale_dags; export_scale_dags('${ARTIFACT_FOLDER}')"
+          - export SOURCE_PATH="workflow_${BUILD_VERSION}.gz"
+          - tar --exclude="bitbucket-pipelines.yml" -cvzf ${SOURCE_PATH} ${ARTIFACT_FOLDER}
+          - export TARGET_PATH="generic-packages/dkist-processing-ops/${BUILD_VERSION}/"
+          - curl -fL https://getcli.jfrog.io | sh
+          - ./jfrog rt u --url $ARTIFACTORY_URL --user $ARTIFACTORY_USER --password $ARTIFACTORY_PASSWORD ${SOURCE_PATH} ${TARGET_PATH}
+    - step: &push_code
+        caches:
+          - pip
+        name: Push Code
+        script:
+          - pip install -U pip
+          - pip install twine build
+          - python -m build --outdir wheelhouse .
+          - python -m twine upload --skip-existing wheelhouse/*
+
+
+
+pipelines:
+  default:
+    - step: *lint
+    - parallel:
+        - step: *scan
+        - step: *test
+  tags:
+    'v*':
+      - parallel:
+          - step: *lint
+          - step: *scan
+          - step: *test
+      - step: *push_workflow
+      - step: *push_code

dkist-processing-ops-1.0.0/dkist_processing_ops/_version.py

@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '1.0.0'
+__version_tuple__ = version_tuple = (1, 0, 0)

dkist-processing-ops-1.0.0/dkist_processing_ops/dags/scale.py

@@ -0,0 +1,78 @@
+"""
+DAG to use up workers to support scaling
+"""
+from os import environ
+from pathlib import Path
+
+from dkist_processing_core.build_utils import export_dags
+
+
+def export_scale_dags(path: Path | str) -> list[Path]:
+    """Export all the ops dags"""
+    result = []
+    dag_prefix = "ops_scale"
+    scales = [16, 32]
+    queues = ["default", "high_memory"]
+    sleep_duration_seconds = 60
+    for queue in queues:
+        for scale in scales:
+            dag_name = f"{dag_prefix}_{queue}_{scale}"
+            dag_body = _scale_dag(
+                dag_name=dag_name,
+                sleep_duration_seconds=sleep_duration_seconds,
+                queue=queue,
+                concurrent_task_count=scale,
+            )
+            dag_path = _export_ops_dag(dag_name=dag_name, dag_body=dag_body, path=path)
+            result.append(dag_path)
+    return result
+
+
+def _export_ops_dag(dag_name: str, dag_body: str, path: Path | str | None = None) -> Path:
+    """Write a file representation of the scaling DAG."""
+    path = path or "dags/"
+    path = Path(path)
+    path.mkdir(exist_ok=True)
+    version = environ.get("BUILD_VERSION", "dev")
+    dag_name = f"{dag_name}_{version}"
+    workflow_py = path / f"{dag_name}.py"
+    with workflow_py.open(mode="w") as f:
+        f.write(dag_body)
+    return workflow_py
+
+
+def _scale_dag(
+    dag_name: str,
+    sleep_duration_seconds: int = 60,
+    queue: str | None = None,
+    concurrent_task_count: int = 16,
+) -> str:
+    queue = queue or "default"
+
+    imports = f"""# Scale {concurrent_task_count} DAG on queue {queue}
+from datetime import timedelta
+import pendulum
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+"""
+    dag = f"""with DAG(
+        dag_id="{dag_name}",
+        start_date=pendulum.today("UTC").add(days=-2),
+        schedule=None,
+        catchup=False,
+        tags=["ops", "scale"],
+    ) as d:"""
+    tasks = []
+    for idx in range(concurrent_task_count):
+        task = f"""    t{idx} = BashOperator(
+            task_id="t{idx}",
+            bash_command=f"sleep {sleep_duration_seconds}",
+            retries=0,
+            retry_delay=timedelta(seconds=60),
+            owner="DKIST Data Center",
+            queue="{queue}",
+        )"""
+        tasks.append(task)
+    parts = [imports, dag] + tasks
+    body = "\n".join(parts)
+    return body

dkist-processing-ops-1.0.0/dkist_processing_ops/tasks/__init__.py

@@ -0,0 +1,2 @@
+"""Import of tasks is directly from package by convention"""
+from dkist_processing_ops.tasks.wait import *

dkist-processing-ops-1.0.0/dkist_processing_ops/tasks/wait.py

@@ -0,0 +1,15 @@
+"""Task for parallelization testing which sleeps a configurable amount of time"""
+from time import sleep
+
+from dkist_processing_core import TaskBase
+
+
+__all__ = ["WaitTask"]
+
+
+SLEEP_TIME = 60
+
+
+class WaitTask(TaskBase):
+    def run(self) -> None:
+        sleep(SLEEP_TIME)

dkist-processing-ops-1.0.0/dkist_processing_ops/tests/test_workflows.py

@@ -0,0 +1,9 @@
+"""Test integrity of workflows."""
+from dkist_processing_core.build_utils import validate_workflows
+
+from dkist_processing_ops import workflows
+
+
+def test_workflow_integrity():
+    """Validate workflow to ensure acyclic-ness and export compilation"""
+    validate_workflows(workflows)

dkist-processing-ops-1.0.0/dkist_processing_ops/workflows/smoke.py

@@ -0,0 +1,28 @@
+"""Workflows to test task submission and spin up"""
+from dkist_processing_common.tasks import TrialTeardown
+from dkist_processing_core import ResourceQueue
+from dkist_processing_core import Workflow
+
+from dkist_processing_ops.tasks import WaitTask
+
+
+smoke_default = Workflow(
+    input_data="ops",
+    output_data="common",
+    category="smoke",
+    detail="default",
+    workflow_package=__package__,
+)
+smoke_default.add_node(task=WaitTask, upstreams=None, resource_queue=ResourceQueue.DEFAULT)
+smoke_default.add_node(task=TrialTeardown, upstreams=WaitTask)
+
+
+smoke_high_mem = Workflow(
+    input_data="ops",
+    output_data="common",
+    category="smoke",
+    detail="high-mem",
+    workflow_package=__package__,
+)
+smoke_high_mem.add_node(task=WaitTask, upstreams=None, resource_queue=ResourceQueue.HIGH_MEMORY)
+smoke_high_mem.add_node(task=TrialTeardown, upstreams=WaitTask)

dkist-processing-ops-1.0.0/dkist_processing_ops.egg-info/PKG-INFO

@@ -0,0 +1,41 @@
+Metadata-Version: 2.1
+Name: dkist-processing-ops
+Version: 1.0.0
+Summary: Automated Processing smoke test and operations workflows
+Author-email: NSO / AURA <dkistdc@nso.edu>
+License: BSD 3-Clause
+Project-URL: repository, https://bitbucket.org/dkistdc/dkist-processing-ops
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.11
+Description-Content-Type: text/x-rst
+License-File: LICENSE.rst
+Requires-Dist: dkist-processing-common==6.1.0
+Requires-Dist: dkist-service-configuration==1.1.0
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: pytest-xdist; extra == "test"
+
+dkist-processing-ops
+--------------------
+|codecov|
+
+This repository works in concert with `dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_ and
+`dkist-processing-common <https://pypi.org/project/dkist-processing-common/>`_ to provide workflows for the
+operational management and smoke testing of the `Automated Processing <https://nso.atlassian.net/wiki/spaces/DPD/pages/3671451/04+-+Automated+Processing>`_ stack.
+
+
+Developer Setup
+~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    pip install -e .[test]
+    pip install pre-commit
+    pre-commit install
+
+
+.. |codecov| image:: https://codecov.io/bb/dkistdc/dkist-processing-ops/branch/main/graph/badge.svg
+   :target: https://codecov.io/bb/dkistdc/dkist-processing-ops

dkist-processing-ops-1.0.0/dkist_processing_ops.egg-info/SOURCES.txt

@@ -0,0 +1,22 @@
+.gitignore
+.pre-commit-config.yaml
+LICENSE.rst
+MANIFEST.in
+README.rst
+bitbucket-pipelines.yml
+pyproject.toml
+dkist_processing_ops/__init__.py
+dkist_processing_ops/_version.py
+dkist_processing_ops.egg-info/PKG-INFO
+dkist_processing_ops.egg-info/SOURCES.txt
+dkist_processing_ops.egg-info/dependency_links.txt
+dkist_processing_ops.egg-info/not-zip-safe
+dkist_processing_ops.egg-info/requires.txt
+dkist_processing_ops.egg-info/top_level.txt
+dkist_processing_ops/dags/scale.py
+dkist_processing_ops/tasks/__init__.py
+dkist_processing_ops/tasks/wait.py
+dkist_processing_ops/tests/__init__.py
+dkist_processing_ops/tests/test_workflows.py
+dkist_processing_ops/workflows/__init__.py
+dkist_processing_ops/workflows/smoke.py

dkist-processing-ops-1.0.0/dkist_processing_ops.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

dkist-processing-ops-1.0.0/dkist_processing_ops.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

dkist-processing-ops-1.0.0/dkist_processing_ops.egg-info/requires.txt

@@ -0,0 +1,7 @@
+dkist-processing-common==6.1.0
+dkist-service-configuration==1.1.0
+
+[test]
+pytest
+pytest-cov
+pytest-xdist

dkist-processing-ops-1.0.0/dkist_processing_ops.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+dkist_processing_ops
+wheelhouse

dkist-processing-ops-1.0.0/pyproject.toml

@@ -0,0 +1,80 @@
+[build-system]
+
+requires = [
+    "setuptools>=62.1",
+    "setuptools_scm[toml]>=6.2",
+]
+
+build-backend = 'setuptools.build_meta'
+
+[project]
+name = "dkist-processing-ops"
+description = "Automated Processing smoke test and operations workflows"
+readme = "README.rst"
+requires-python = ">=3.11"
+classifiers = [
+    "License :: OSI Approved :: BSD License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+]
+license = { text = "BSD 3-Clause" }
+authors = [
+  { name = "NSO / AURA", email = "dkistdc@nso.edu" },
+]
+
+dependencies = [
+    "dkist-processing-common==6.1.0",
+    "dkist-service-configuration==1.1.0",
+]
+dynamic = ["version"]
+
+# tox is not required to run the tests, but simplifies IDE integration
+# Pygments is solely to support README.rst rendering
+[project.optional-dependencies]
+test = [
+    "pytest",
+    "pytest-cov",
+    "pytest-xdist",
+]
+
+[project.urls]
+repository = "https://bitbucket.org/dkistdc/dkist-processing-ops"
+
+[tool.setuptools]
+zip-safe = false
+include-package-data = true
+
+[tool.setuptools.packages.find]
+
+[tool.setuptools.package-data]
+# include .ttf files from any directory
+"*" = ["*.ttf"]
+
+[tool.setuptools_scm]
+write_to = "dkist_processing_ops/_version.py"
+
+[tool.pytest.ini_options]
+testpaths = [
+  "dkist_processing_ops",
+]
+markers = [
+  "development: For tests that can only be  run while developing with a sidecar proxy (as opposed to in bitbucket pipelines)"
+]
+
+[tool.coverage.run]
+omit = [
+ "dkist_processing_ops/__init*",
+ "dkist_processing_ops/conftest.py",
+ "dkist_processing_ops/*setup_package*",
+ "dkist_processing_ops/tests/*",
+ "dkist_processing_ops/*/tests/*",
+ "dkist_processing_ops/extern/*",
+ "dkist_processing_ops/version*",
+ "*/dkist_processing_ops/__init*",
+ "*/dkist_processing_ops/conftest.py",
+ "*/dkist_processing_ops/*setup_package*",
+ "*/dkist_processing_ops/tests/*",
+ "*/dkist_processing_ops/*/tests/*",
+ "*/dkist_processing_ops/extern/*",
+ "*/dkist_processing_ops/version*",
+]

dkist-processing-ops-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+