dkist-processing-dlnirsp 0.1.0__tar.gz

dkist_processing_dlnirsp-0.1.0/.gitignore

@@ -0,0 +1,194 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+__pycache__
+*.py[cod]
+*$py.class
+*.a
+*.o
+*.pyc
+
+# C extensions
+*.so
+
+# Distribution / packaging
+*.egg
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+distribute-*.tar.gz
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+MANIFEST
+scratch
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+docs/api/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# PyCharm project settings
+.idea
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+
+# other
+*.npy
+*.dat
+*.fits
+.tox
+.*.sw[op]
+*~
+
+# Ignore .c files by default to avoid including generated code. If you want to
+# add a non-generated .c extension, use `git add -f filename.c`.
+*.c
+
+# Other generated files
+*/version.py
+*/cython_version.py
+htmlcov
+
+
+# Sphinx
+docs/api
+docs/_build
+
+# Eclipse editor project files
+.project
+.pydevproject
+.settings
+
+# Floobits project files
+.floo
+.flooignore
+
+# Mac OSX
+.DS_Store
+
+# Outputs and temp files
+*.png
+*.txt
+*.cprof
+vtfcal/dask-worker-space/
+vtfcal/data/
+tmp??_pars*.fits
+
+*data_tree
+dask-worker-space
+logging.*
+
+# VS Code
+.history/
+.vscode/

dkist_processing_dlnirsp-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.4.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+        args: [--maxkb=700]
+    -   id: debug-statements
+-   repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+    -   id: black
+        args: [ "-l 100" ]
+-   repo: https://github.com/asottile/reorder_python_imports
+    rev: v2.3.5
+    hooks:
+    -   id: reorder-python-imports
+-   repo: https://github.com/pycqa/pydocstyle
+    rev: 6.1.1  # pick a git hash / tag to point to
+    hooks:
+    -   id: pydocstyle
+        exclude: test
+        args:
+            - --convention=numpy
+            - --add-ignore=D105

dkist_processing_dlnirsp-0.1.0/.readthedocs.yml

@@ -0,0 +1,21 @@
+version: 2
+build:
+  os: ubuntu-20.04
+  tools:
+    python: "3.11"
+  apt_packages:
+    - libopenjp2-7
+    - graphviz
+
+sphinx:
+  builder: html
+  configuration: docs/conf.py
+  fail_on_warning: false
+
+python:
+   install:
+     - method: pip
+       extra_requirements:
+        - all
+        - docs
+       path: .

dkist_processing_dlnirsp-0.1.0/CHANGELOG.rst

@@ -0,0 +1,4 @@
+v0.1.0 (2024-06-06)
+===================
+
+- Initial release. Mostly for first release to DC stacks (i.e., not "production" quality).

dkist_processing_dlnirsp-0.1.0/PKG-INFO

@@ -0,0 +1,118 @@
+Metadata-Version: 2.1
+Name: dkist_processing_dlnirsp
+Version: 0.1.0
+Summary: Science processing code for the DLNIRSP instrument on DKIST
+Home-page: https://bitbucket.org/dkistdc/dkist_processing_dlnirsp/src/main/
+Author: NSO / AURA
+Author-email: "dkistdc@nso.edu"
+License: MIT
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.11
+Provides-Extra: test
+Provides-Extra: inventory
+Provides-Extra: asdf
+Provides-Extra: quality
+Provides-Extra: grogu
+Provides-Extra: docs
+
+dkist-processing-dlnirsp
+========================
+
+Overview
+--------
+The dkist-processing-dlnirsp library contains the implementation of the DLNIRSP pipelines as a collection of the
+`dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_ framework and
+`dkist-processing-common <https://pypi.org/project/dkist-processing-common/>`_ Tasks.
+
+The recommended project structure is to separate tasks and workflows into seperate packages.  Having the workflows
+in their own package facilitates using the build_utils to test the integrity of those workflows in the unit test.
+
+Calibration Pipeline
+------------------------
+
+Build
+--------
+Artifacts are built through `bitbucket pipelines <bitbucket-pipelines.yml>`_.
+
+The pipeline can be used in other repos with a modification of the package and artifact locations
+to use the names relevant to the target repo.
+
+e.g. dkist-processing-test -> dkist-processing-vbi and dkist_processing_test -> dkist_processing_vbi
+
+Deployment
+----------
+Deployment is done with `turtlebot <https://bitbucket.org/dkistdc/turtlebot/src/master/>`_ and follows
+the process detailed in `dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_
+
+Environment Variables
+---------------------
+Only those specified by `dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_ and `dkist-processing-common <https://pypi.org/project/dkist-processing-common/>`_.
+
+Development
+-----------
+.. code-block:: bash
+
+    git clone git@bitbucket.org:dkistdc/dkist-processing-dlnirsp.git
+    cd dkist-processing-dlnirsp
+    pre-commit install
+    pip install -e .[test]
+    pytest -v --cov dkist_processing_nirsp
+
+Changelog
+#########
+
+When you make **any** change to this repository it **MUST** be accompanied by a changelog file.
+The changelog for this repository uses the `towncrier <https://github.com/twisted/towncrier>`__ package.
+Entries in the changelog for the next release are added as individual files (one per change) to the ``changelog/`` directory.
+
+Writing a Changelog Entry
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A changelog entry accompanying a change should be added to the ``changelog/`` directory.
+The name of a file in this directory follows a specific template::
+
+  <PULL REQUEST NUMBER>.<TYPE>[.<COUNTER>].rst
+
+The fields have the following meanings:
+
+* ``<PULL REQUEST NUMBER>``: This is the number of the pull request, so people can jump from the changelog entry to the diff on BitBucket.
+* ``<TYPE>``: This is the type of the change and must be one of the values described below.
+* ``<COUNTER>``: This is an optional field, if you make more than one change of the same type you can append a counter to the subsequent changes, i.e. ``100.bugfix.rst`` and ``100.bugfix.1.rst`` for two bugfix changes in the same PR.
+
+The list of possible types is defined the the towncrier section of ``pyproject.toml``, the types are:
+
+* ``feature``: This change is a new code feature.
+* ``bugfix``: This is a change which fixes a bug.
+* ``doc``: A documentation change.
+* ``removal``: A deprecation or removal of public API.
+* ``misc``: Any small change which doesn't fit anywhere else, such as a change to the package infrastructure.
+
+
+Rendering the Changelog at Release Time
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When you are about to tag a release first you must run ``towncrier`` to render the changelog.
+The steps for this are as follows:
+
+* Run `towncrier build --version vx.y.z` using the version number you want to tag.
+* Agree to have towncrier remove the fragments.
+* Add and commit your changes.
+* Tag the release.
+
+**NOTE:** If you forget to add a Changelog entry to a tagged release (either manually or automatically with ``towncrier``)
+then the Bitbucket pipeline will fail. To be able to use the same tag you must delete it locally and on the remote branch:
+
+.. code-block:: bash
+
+    # First, actually update the CHANGELOG and commit the update
+    git commit
+
+    # Delete tags
+    git tag -d vWHATEVER.THE.VERSION
+    git push --delete origin vWHATEVER.THE.VERSION
+
+    # Re-tag with the same version
+    git tag vWHATEVER.THE.VERSION
+    git push --tags origin main

dkist_processing_dlnirsp-0.1.0/README.rst

@@ -0,0 +1,99 @@
+dkist-processing-dlnirsp
+========================
+
+Overview
+--------
+The dkist-processing-dlnirsp library contains the implementation of the DLNIRSP pipelines as a collection of the
+`dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_ framework and
+`dkist-processing-common <https://pypi.org/project/dkist-processing-common/>`_ Tasks.
+
+The recommended project structure is to separate tasks and workflows into seperate packages.  Having the workflows
+in their own package facilitates using the build_utils to test the integrity of those workflows in the unit test.
+
+Calibration Pipeline
+------------------------
+
+Build
+--------
+Artifacts are built through `bitbucket pipelines <bitbucket-pipelines.yml>`_.
+
+The pipeline can be used in other repos with a modification of the package and artifact locations
+to use the names relevant to the target repo.
+
+e.g. dkist-processing-test -> dkist-processing-vbi and dkist_processing_test -> dkist_processing_vbi
+
+Deployment
+----------
+Deployment is done with `turtlebot <https://bitbucket.org/dkistdc/turtlebot/src/master/>`_ and follows
+the process detailed in `dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_
+
+Environment Variables
+---------------------
+Only those specified by `dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_ and `dkist-processing-common <https://pypi.org/project/dkist-processing-common/>`_.
+
+Development
+-----------
+.. code-block:: bash
+
+    git clone git@bitbucket.org:dkistdc/dkist-processing-dlnirsp.git
+    cd dkist-processing-dlnirsp
+    pre-commit install
+    pip install -e .[test]
+    pytest -v --cov dkist_processing_nirsp
+
+Changelog
+#########
+
+When you make **any** change to this repository it **MUST** be accompanied by a changelog file.
+The changelog for this repository uses the `towncrier <https://github.com/twisted/towncrier>`__ package.
+Entries in the changelog for the next release are added as individual files (one per change) to the ``changelog/`` directory.
+
+Writing a Changelog Entry
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A changelog entry accompanying a change should be added to the ``changelog/`` directory.
+The name of a file in this directory follows a specific template::
+
+  <PULL REQUEST NUMBER>.<TYPE>[.<COUNTER>].rst
+
+The fields have the following meanings:
+
+* ``<PULL REQUEST NUMBER>``: This is the number of the pull request, so people can jump from the changelog entry to the diff on BitBucket.
+* ``<TYPE>``: This is the type of the change and must be one of the values described below.
+* ``<COUNTER>``: This is an optional field, if you make more than one change of the same type you can append a counter to the subsequent changes, i.e. ``100.bugfix.rst`` and ``100.bugfix.1.rst`` for two bugfix changes in the same PR.
+
+The list of possible types is defined the the towncrier section of ``pyproject.toml``, the types are:
+
+* ``feature``: This change is a new code feature.
+* ``bugfix``: This is a change which fixes a bug.
+* ``doc``: A documentation change.
+* ``removal``: A deprecation or removal of public API.
+* ``misc``: Any small change which doesn't fit anywhere else, such as a change to the package infrastructure.
+
+
+Rendering the Changelog at Release Time
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When you are about to tag a release first you must run ``towncrier`` to render the changelog.
+The steps for this are as follows:
+
+* Run `towncrier build --version vx.y.z` using the version number you want to tag.
+* Agree to have towncrier remove the fragments.
+* Add and commit your changes.
+* Tag the release.
+
+**NOTE:** If you forget to add a Changelog entry to a tagged release (either manually or automatically with ``towncrier``)
+then the Bitbucket pipeline will fail. To be able to use the same tag you must delete it locally and on the remote branch:
+
+.. code-block:: bash
+
+    # First, actually update the CHANGELOG and commit the update
+    git commit
+
+    # Delete tags
+    git tag -d vWHATEVER.THE.VERSION
+    git push --delete origin vWHATEVER.THE.VERSION
+
+    # Re-tag with the same version
+    git tag vWHATEVER.THE.VERSION
+    git push --tags origin main

dkist_processing_dlnirsp-0.1.0/SCIENCE_CHANGELOG.rst

@@ -0,0 +1,5 @@
+v0.1.0 (2024-06-06)
+===================
+
+- Initial release. Pipeline supports both BIFOS and MISI data and produces valid L1 frames. IFU-remapping is not yet implemented
+  so the L1 files are presented as a single slit. WCS header values not guaranteed.

dkist_processing_dlnirsp-0.1.0/bitbucket-pipelines.yml

@@ -0,0 +1,110 @@
+#Build Configuration for docker deployment to artifactory
+image: python:3.11
+
+definitions:
+  services:
+    redis:
+      image: redis
+  steps:
+    - step: &lint
+        caches:
+          - pip
+        name: Lint
+        script:
+          - pip install pre-commit
+          - pre-commit install
+          - pre-commit run --all-files
+    - step: &changelog
+        caches:
+          - pip
+        name: Changelog
+        script:
+          - pip install towncrier
+          # We need to fetch the main branch to compare against
+          - git remote set-branches --add origin main
+          - git fetch origin main
+          - towncrier check --compare-with origin/main
+    - step: &scan
+        caches:
+          - pip
+        name: Scan
+        script:
+          - pip install -e .
+          - pip freeze | grep -v @ > requirements.txt
+          - cat requirements.txt
+          - echo $SNYK_VERSION
+          - curl -L -o snyk https://github.com/snyk/snyk/releases/download/$SNYK_VERSION/snyk-linux
+          - chmod 755 snyk
+          - ./snyk -d auth $SNYK_TOKEN
+          - echo $SNYK_IGNORE
+          - for id in $SNYK_IGNORE; do echo Ignoring $id; ./snyk ignore $id; done
+          - cat .snyk || echo "No .snyk found. Probably because there was nothing to ignore."
+          - echo $SNYK_CLI_COMMAND
+          - $SNYK_CLI_COMMAND
+    - step: &test
+        caches:
+          - pip
+        name: Test
+        script:
+          - pip install .[test]
+          - pytest -v -n auto --dist worksteal --cov -m "not development" dkist_processing_dlnirsp
+        services:
+          - redis
+    - step: &docs
+        name: Test Docs
+        caches:
+          - pip
+        script:
+          - apt update && apt -y install graphviz
+          - pip install .[docs]
+          - sphinx-build --color -W --keep-going -b html docs docs/_build/html
+    - step: &check_changelog
+        name: Check for updated CHANGELOG
+        script:
+          - ./check_changelog_updated.sh
+    - step: &push_workflow
+        caches:
+          - pip
+        name: Push Workflow
+        script:
+          - pip install .
+          - export BUILD_VERSION="${BITBUCKET_TAG:1}"
+          - export ARTIFACT_FOLDER="${BITBUCKET_REPO_SLUG}_${BUILD_VERSION}/"
+          - python -c "from dkist_processing_core.build_utils import export_dags; import dkist_processing_dlnirsp.workflows as workflow_package; export_dags(workflow_package, '${ARTIFACT_FOLDER}')"
+          - export SOURCE_PATH="workflow_${BUILD_VERSION}.gz"
+          - tar --exclude="bitbucket-pipelines.yml" -cvzf ${SOURCE_PATH} ${ARTIFACT_FOLDER}
+          - export TARGET_PATH="generic-packages/dkist-processing-dlnirsp/${BUILD_VERSION}/"
+          - curl -fL https://getcli.jfrog.io | sh
+          - ./jfrog rt u --url $ARTIFACTORY_URL --user $ARTIFACTORY_USER --password $ARTIFACTORY_PASSWORD ${SOURCE_PATH} ${TARGET_PATH}
+    - step: &push_code
+        caches:
+          - pip
+        name: Push Code
+        script:
+          - python setup.py sdist
+          - pip install twine
+          - twine upload dist/*
+
+
+options:
+  max-time: 25
+pipelines:
+  default:
+    - parallel:
+      - step: *lint
+      - step: *changelog
+    - parallel:
+      - step: *scan
+      - step: *test
+      - step: *docs
+  tags:
+    'v*':
+      - parallel:
+        - step: *lint
+        - step: *check_changelog
+      - parallel:
+        - step: *test
+        - step: *scan
+        - step: *docs
+      - step: *push_workflow
+      - step: *push_code

dkist_processing_dlnirsp-0.1.0/check_changelog_updated.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+CURRENT_VERSION=`git describe --tags --match v* --exact-match`
+ERROR_STR="CHANGELOG has not been updated for ${CURRENT_VERSION}\\nSee the README for instructions"
+ERROR_PAD="***********"
+
+if [[ $CURRENT_VERSION =~ "rc" ]]; then
+  echo "release candidate detected"
+  exit 0;
+fi
+
+ls changelog/*rst 2>/dev/null && {
+  echo -e "\\n${ERROR_PAD}"
+  echo -e "Unmerged changelog fragments found! You must run towncrier!\\nSee the README for instruction"
+  echo -e "${ERROR_PAD}"
+  exit 1;
+}
+
+head -n 1 CHANGELOG.rst | grep -q $CURRENT_VERSION - || {
+  echo -e "\\n${ERROR_PAD}"
+  echo -e $ERROR_STR;
+  echo -e "${ERROR_PAD}"
+  exit 1; }

dkist_processing_dlnirsp-0.1.0/dkist_processing_dlnirsp/__init__.py

@@ -0,0 +1,9 @@
+"""Processing pipeline for DLNIRSP instrument on DKIST."""
+from pkg_resources import DistributionNotFound
+from pkg_resources import get_distribution
+
+try:
+    __version__ = get_distribution(__name__).version
+except DistributionNotFound:
+    # package is not installed
+    __version__ = "unknown"

dkist_processing_dlnirsp-0.1.0/dkist_processing_dlnirsp/config.py

@@ -0,0 +1,12 @@
+"""Configuration for the dkist-processing-dlnirsp package and logging thereof."""
+from dkist_processing_common.config import DKISTProcessingCommonConfiguration
+
+
+class DKISTProcessingDLNIRSPConfigurations(DKISTProcessingCommonConfiguration):
+    """Configurations custom to the dkist-processing-dlnirsp package."""
+
+    pass  # nothing custom.... yet
+
+
+dkist_processing_dlnirsp_configurations = DKISTProcessingDLNIRSPConfigurations()
+dkist_processing_dlnirsp_configurations.log_configurations()

dkist_processing_dlnirsp-0.1.0/dkist_processing_dlnirsp/dev_scripts/__init__.py

@@ -0,0 +1 @@
+"""Modules to allow easy testing of pipeline components by pipeline developers."""