dkist-processing-core 4.2.1__tar.gz → 5.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dkist_processing_core-5.0.0/.snyk +3 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/CHANGELOG.rst +35 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/PKG-INFO +33 -7
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/bitbucket-pipelines.yml +25 -30
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/node.py +1 -1
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/test_node.py +22 -3
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/workflow.py +1 -1
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core.egg-info/PKG-INFO +33 -7
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core.egg-info/SOURCES.txt +2 -3
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core.egg-info/requires.txt +1 -1
- dkist_processing_core-5.0.0/dkist_processing_core.egg-info/top_level.txt +6 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/index.rst +1 -1
- dkist_processing_core-5.0.0/docs/landing_page.rst +34 -0
- dkist_processing_core-5.0.0/pyproject.toml +109 -0
- dkist_processing_core-5.0.0/setup.cfg +4 -0
- dkist-processing-core-4.2.1/check_changelog_updated.sh +0 -16
- dkist-processing-core-4.2.1/dkist_processing_core.egg-info/top_level.txt +0 -1
- dkist-processing-core-4.2.1/pyproject.toml +0 -32
- dkist-processing-core-4.2.1/setup.cfg +0 -73
- dkist-processing-core-4.2.1/setup.py +0 -4
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/.gitignore +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/.pre-commit-config.yaml +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/.readthedocs.yml +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/README.rst +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/changelog/.gitempty +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/__init__.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/build_utils.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/config.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/failure_callback.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/resource_queue.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/task.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/__init__.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/conftest.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/invalid_workflow_cyclic/__init__.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/invalid_workflow_cyclic/workflow.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/invalid_workflow_for_docker_multi_category/__init__.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/invalid_workflow_for_docker_multi_category/workflow.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/task_example.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/test_build_utils.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/test_export.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/test_failure_callback.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/test_task.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/test_workflow.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/valid_workflow_package/__init__.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/valid_workflow_package/workflow.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/zero_node_workflow_package/__init__.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/zero_node_workflow_package/workflow.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core.egg-info/dependency_links.txt +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/Makefile +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/auto-proc-concept-model.png +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/auto_proc_brick.png +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/automated-processing-deployed.png +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/changelog.rst +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/conf.py +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/make.bat +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/requirements.txt +0 -0
- {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/licenses/LICENSE.rst +0 -0
|
@@ -1,3 +1,38 @@
|
|
|
1
|
+
v5.0.0 (2025-02-14)
|
|
2
|
+
===================
|
|
3
|
+
|
|
4
|
+
Features
|
|
5
|
+
--------
|
|
6
|
+
|
|
7
|
+
- Instrument python packages are now installed using their "frozen" pip extra, which contains a complete set of frozen dependencies.
|
|
8
|
+
This ensures the environment is exactly the same for a given version of an instrument repo. (`#48 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/48>`__)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
Misc
|
|
12
|
+
----
|
|
13
|
+
|
|
14
|
+
- Update bitbucket pipelines to use common scripts for checking for changelog snippets and verifying doc builds. (`#47 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/47>`__)
|
|
15
|
+
- Update Bitbucket pipelines to use execute script for standard steps. (`#49 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/49>`__)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
v4.3.0 (2025-01-08)
|
|
19
|
+
===================
|
|
20
|
+
|
|
21
|
+
Misc
|
|
22
|
+
----
|
|
23
|
+
|
|
24
|
+
- Make and publish wheels at code push in build pipeline (`#43 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/43>`__)
|
|
25
|
+
- Switch from setup.cfg to pyproject.toml for build configuration (`#43 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/43>`__)
|
|
26
|
+
- Update Bitbucket pipelines to use standardized lint and scan steps. (`#45 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/45>`__)
|
|
27
|
+
- Upgrade apache-airflow to 2.10.4. (`#46 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/46>`__)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
Documentation
|
|
31
|
+
-------------
|
|
32
|
+
|
|
33
|
+
- Change the documentation landing page to focus more on users and less on developers. (`#44 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/44>`__)
|
|
34
|
+
|
|
35
|
+
|
|
1
36
|
v4.2.1 (2024-09-27)
|
|
2
37
|
===================
|
|
3
38
|
|
|
@@ -1,18 +1,44 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: dkist-processing-core
|
|
3
|
-
Version:
|
|
4
|
-
Summary: Abstraction layer
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
3
|
+
Version: 5.0.0
|
|
4
|
+
Summary: Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow
|
|
5
|
+
Author-email: NSO / AURA <dkistdc@nso.edu>
|
|
6
|
+
License: BSD-3-Clause
|
|
7
|
+
Project-URL: Homepage, https://nso.edu/dkist/data-center/
|
|
8
|
+
Project-URL: Repository, https://bitbucket.org/dkistdc/dkist-processing-core/
|
|
9
9
|
Project-URL: Documentation, https://docs.dkist.nso.edu/projects/core
|
|
10
|
+
Project-URL: Help, https://nso.atlassian.net/servicedesk/customer/portal/5
|
|
10
11
|
Classifier: Programming Language :: Python
|
|
11
12
|
Classifier: Programming Language :: Python :: 3
|
|
12
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
14
|
Requires-Python: >=3.11
|
|
15
|
+
Description-Content-Type: text/x-rst
|
|
16
|
+
Requires-Dist: apache-airflow[celery,postgres]==2.10.4
|
|
17
|
+
Requires-Dist: elastic-apm<7.0.0
|
|
18
|
+
Requires-Dist: requests>=2.23
|
|
19
|
+
Requires-Dist: talus<2.0,>=1.1.0
|
|
20
|
+
Requires-Dist: pendulum
|
|
21
|
+
Requires-Dist: nbformat>=5.9.2
|
|
22
|
+
Requires-Dist: dkist-service-configuration<3.0,>=2.0.2
|
|
23
|
+
Requires-Dist: pydantic>2.0
|
|
14
24
|
Provides-Extra: test
|
|
25
|
+
Requires-Dist: pytest; extra == "test"
|
|
26
|
+
Requires-Dist: pytest-cov; extra == "test"
|
|
27
|
+
Requires-Dist: pytest-mock; extra == "test"
|
|
28
|
+
Requires-Dist: jinja2; extra == "test"
|
|
29
|
+
Requires-Dist: towncrier; extra == "test"
|
|
30
|
+
Requires-Dist: nbconvert; extra == "test"
|
|
31
|
+
Requires-Dist: ipython; extra == "test"
|
|
15
32
|
Provides-Extra: docs
|
|
33
|
+
Requires-Dist: sphinx; extra == "docs"
|
|
34
|
+
Requires-Dist: sphinx-astropy; extra == "docs"
|
|
35
|
+
Requires-Dist: sphinx-changelog; extra == "docs"
|
|
36
|
+
Requires-Dist: sphinx-autoapi; extra == "docs"
|
|
37
|
+
Requires-Dist: pytest; extra == "docs"
|
|
38
|
+
Requires-Dist: towncrier; extra == "docs"
|
|
39
|
+
Requires-Dist: dkist-sphinx-theme; extra == "docs"
|
|
40
|
+
Requires-Dist: matplotlib; extra == "docs"
|
|
41
|
+
Requires-Dist: sunpy; extra == "docs"
|
|
16
42
|
|
|
17
43
|
dkist-processing-core
|
|
18
44
|
=====================
|
|
@@ -6,41 +6,34 @@ definitions:
|
|
|
6
6
|
ports:
|
|
7
7
|
- 5672:5672
|
|
8
8
|
steps:
|
|
9
|
+
- step: &download_execute_script
|
|
10
|
+
name: Download Script Execute Script
|
|
11
|
+
script:
|
|
12
|
+
- echo "Retrieving Script Execution Script"
|
|
13
|
+
- curl -fL https://getcli.jfrog.io | sh
|
|
14
|
+
- ./jfrog rt dl --url $ARTIFACTORY_URL --user $ARTIFACTORY_USER --password $ARTIFACTORY_PASSWORD generic-packages/ci_scripts/latest/execute_script.sh
|
|
15
|
+
- cat ci_scripts/latest/execute_script.sh
|
|
16
|
+
- chmod 755 ci_scripts/latest/execute_script.sh
|
|
17
|
+
artifacts:
|
|
18
|
+
- ci_scripts/latest/execute_script.sh
|
|
9
19
|
- step: &lint
|
|
10
20
|
caches:
|
|
11
21
|
- pip
|
|
12
22
|
name: Lint
|
|
13
23
|
script:
|
|
14
|
-
-
|
|
15
|
-
- pre-commit install
|
|
16
|
-
- pre-commit run --all-files
|
|
24
|
+
- ./ci_scripts/latest/execute_script.sh lint_python.sh
|
|
17
25
|
- step: &changelog
|
|
18
26
|
caches:
|
|
19
27
|
- pip
|
|
20
28
|
name: Changelog
|
|
21
29
|
script:
|
|
22
|
-
-
|
|
23
|
-
# We need to fetch the main branch to compare against
|
|
24
|
-
- git remote set-branches --add origin main
|
|
25
|
-
- git fetch origin main
|
|
26
|
-
- towncrier check --compare-with origin/main
|
|
30
|
+
- ./ci_scripts/latest/execute_script.sh check_changelog.sh
|
|
27
31
|
- step: &scan
|
|
28
32
|
caches:
|
|
29
33
|
- pip
|
|
30
34
|
name: Scan
|
|
31
35
|
script:
|
|
32
|
-
-
|
|
33
|
-
- pip freeze | grep -v @ > requirements.txt
|
|
34
|
-
- cat requirements.txt
|
|
35
|
-
- echo $SNYK_VERSION
|
|
36
|
-
- curl -L -o snyk https://github.com/snyk/snyk/releases/download/$SNYK_VERSION/snyk-linux
|
|
37
|
-
- chmod 755 snyk
|
|
38
|
-
- ./snyk -d auth $SNYK_TOKEN
|
|
39
|
-
- echo $SNYK_IGNORE
|
|
40
|
-
- for id in $SNYK_IGNORE; do echo Ignoring $id; ./snyk ignore $id; done
|
|
41
|
-
- cat .snyk || echo "No .snyk found. Probably because there was nothing to ignore."
|
|
42
|
-
- echo $SNYK_CLI_COMMAND
|
|
43
|
-
- $SNYK_CLI_COMMAND
|
|
36
|
+
- ./ci_scripts/latest/execute_script.sh scan_python.sh
|
|
44
37
|
- step: &test
|
|
45
38
|
caches:
|
|
46
39
|
- pip
|
|
@@ -52,19 +45,12 @@ definitions:
|
|
|
52
45
|
script:
|
|
53
46
|
- pip install .[test]
|
|
54
47
|
- pytest -m "not development" --cov dkist_processing_core
|
|
55
|
-
|
|
56
48
|
- step: &push
|
|
57
49
|
caches:
|
|
58
50
|
- pip
|
|
59
51
|
name: Push
|
|
60
52
|
script:
|
|
61
|
-
-
|
|
62
|
-
- pip install twine
|
|
63
|
-
- twine upload dist/*
|
|
64
|
-
- step: &check_changelog
|
|
65
|
-
name: Check for updated CHANGELOG
|
|
66
|
-
script:
|
|
67
|
-
- ./check_changelog_updated.sh
|
|
53
|
+
- ./ci_scripts/latest/execute_script.sh push_python.sh
|
|
68
54
|
- step: &build_mpw
|
|
69
55
|
caches:
|
|
70
56
|
- pip
|
|
@@ -79,24 +65,33 @@ definitions:
|
|
|
79
65
|
- cat Dockerfile
|
|
80
66
|
- docker login dkistdc-docker.jfrog.io --username $DOCKER_USERNAME --password $DOCKER_PASSWORD
|
|
81
67
|
- docker build -t $DOCKER_HUB_URL/${BITBUCKET_REPO_SLUG}:latest .
|
|
68
|
+
- step: &docs
|
|
69
|
+
name: Test Docs
|
|
70
|
+
caches:
|
|
71
|
+
- pip
|
|
72
|
+
script:
|
|
73
|
+
- ./ci_scripts/latest/execute_script.sh test_sphinx.sh
|
|
82
74
|
|
|
83
75
|
pipelines:
|
|
84
76
|
default:
|
|
77
|
+
- step: *download_execute_script
|
|
85
78
|
- parallel:
|
|
86
79
|
- step: *lint
|
|
87
80
|
- step: *changelog
|
|
88
81
|
- parallel:
|
|
89
82
|
- step: *scan
|
|
90
83
|
- step: *test
|
|
84
|
+
- step: *docs
|
|
91
85
|
- step: *build_mpw
|
|
92
|
-
|
|
93
86
|
tags:
|
|
94
87
|
'v*':
|
|
88
|
+
- step: *download_execute_script
|
|
95
89
|
- parallel:
|
|
96
|
-
- step: *
|
|
90
|
+
- step: *changelog
|
|
97
91
|
- step: *lint
|
|
98
92
|
- parallel:
|
|
99
93
|
- step: *scan
|
|
100
94
|
- step: *test
|
|
95
|
+
- step: *docs
|
|
101
96
|
- step: *build_mpw
|
|
102
97
|
- step: *push
|
|
@@ -44,7 +44,7 @@ class Node:
|
|
|
44
44
|
self.workflow_package = workflow_package
|
|
45
45
|
self.upstreams = upstreams
|
|
46
46
|
self.resource_queue = resource_queue
|
|
47
|
-
self.pip_extras = pip_extras
|
|
47
|
+
self.pip_extras = ["frozen"] + (pip_extras or [])
|
|
48
48
|
|
|
49
49
|
@property
|
|
50
50
|
def operator(self) -> BashOperator:
|
{dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/test_node.py
RENAMED
|
@@ -13,11 +13,16 @@ from dkist_processing_core.tests.task_example import Task
|
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
@pytest.fixture()
|
|
16
|
-
def
|
|
16
|
+
def single_node_version() -> str:
|
|
17
|
+
return "3.4.5"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@pytest.fixture()
|
|
21
|
+
def single_node(pip_extras, single_node_version) -> Node:
|
|
17
22
|
"""A single Node instance."""
|
|
18
23
|
return Node(
|
|
19
24
|
workflow_name="test_node",
|
|
20
|
-
workflow_version=
|
|
25
|
+
workflow_version=single_node_version,
|
|
21
26
|
workflow_package=__package__,
|
|
22
27
|
task=Task,
|
|
23
28
|
resource_queue=ResourceQueue.DEFAULT,
|
|
@@ -45,7 +50,21 @@ def test_nodes(node, fake_producer_factory, queue_name, pip_extras):
|
|
|
45
50
|
assert node.task == task
|
|
46
51
|
assert node.workflow_version == version
|
|
47
52
|
assert node.resource_queue == queue_name
|
|
48
|
-
assert node.pip_extras == pip_extras
|
|
53
|
+
assert node.pip_extras == ["frozen"] + (pip_extras or [])
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_node_install_command(single_node, pip_extras, single_node_version):
|
|
57
|
+
"""
|
|
58
|
+
Given: A valid node instance.
|
|
59
|
+
When: Creating the pip install commands with and without pip extras
|
|
60
|
+
Then: The correct command is generated; pip extras are installed separately from the "frozen" extra
|
|
61
|
+
"""
|
|
62
|
+
expected_repo_name = __package__.split(".")[0].replace("_", "-")
|
|
63
|
+
extras_str = f"'[{','.join(['frozen'] + (pip_extras or []))}]'"
|
|
64
|
+
expected_install_command = f"""python -m pip install --upgrade pip
|
|
65
|
+
python -m pip install {expected_repo_name}{extras_str}=={single_node_version}"""
|
|
66
|
+
|
|
67
|
+
assert single_node.install_command == expected_install_command
|
|
49
68
|
|
|
50
69
|
|
|
51
70
|
@pytest.mark.long()
|
{dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/workflow.py
RENAMED
|
@@ -7,11 +7,11 @@ from pathlib import Path
|
|
|
7
7
|
import nbformat as nbf
|
|
8
8
|
from airflow import DAG
|
|
9
9
|
|
|
10
|
-
from dkist_processing_core import ResourceQueue
|
|
11
10
|
from dkist_processing_core.config import core_configurations
|
|
12
11
|
from dkist_processing_core.node import Node
|
|
13
12
|
from dkist_processing_core.node import task_type_hint
|
|
14
13
|
from dkist_processing_core.node import upstreams_type_hint
|
|
14
|
+
from dkist_processing_core.resource_queue import ResourceQueue
|
|
15
15
|
|
|
16
16
|
__all__ = ["Workflow", "workflow_name_from_details"]
|
|
17
17
|
|
{dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core.egg-info/PKG-INFO
RENAMED
|
@@ -1,18 +1,44 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: dkist-processing-core
|
|
3
|
-
Version:
|
|
4
|
-
Summary: Abstraction layer
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
3
|
+
Version: 5.0.0
|
|
4
|
+
Summary: Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow
|
|
5
|
+
Author-email: NSO / AURA <dkistdc@nso.edu>
|
|
6
|
+
License: BSD-3-Clause
|
|
7
|
+
Project-URL: Homepage, https://nso.edu/dkist/data-center/
|
|
8
|
+
Project-URL: Repository, https://bitbucket.org/dkistdc/dkist-processing-core/
|
|
9
9
|
Project-URL: Documentation, https://docs.dkist.nso.edu/projects/core
|
|
10
|
+
Project-URL: Help, https://nso.atlassian.net/servicedesk/customer/portal/5
|
|
10
11
|
Classifier: Programming Language :: Python
|
|
11
12
|
Classifier: Programming Language :: Python :: 3
|
|
12
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
14
|
Requires-Python: >=3.11
|
|
15
|
+
Description-Content-Type: text/x-rst
|
|
16
|
+
Requires-Dist: apache-airflow[celery,postgres]==2.10.4
|
|
17
|
+
Requires-Dist: elastic-apm<7.0.0
|
|
18
|
+
Requires-Dist: requests>=2.23
|
|
19
|
+
Requires-Dist: talus<2.0,>=1.1.0
|
|
20
|
+
Requires-Dist: pendulum
|
|
21
|
+
Requires-Dist: nbformat>=5.9.2
|
|
22
|
+
Requires-Dist: dkist-service-configuration<3.0,>=2.0.2
|
|
23
|
+
Requires-Dist: pydantic>2.0
|
|
14
24
|
Provides-Extra: test
|
|
25
|
+
Requires-Dist: pytest; extra == "test"
|
|
26
|
+
Requires-Dist: pytest-cov; extra == "test"
|
|
27
|
+
Requires-Dist: pytest-mock; extra == "test"
|
|
28
|
+
Requires-Dist: jinja2; extra == "test"
|
|
29
|
+
Requires-Dist: towncrier; extra == "test"
|
|
30
|
+
Requires-Dist: nbconvert; extra == "test"
|
|
31
|
+
Requires-Dist: ipython; extra == "test"
|
|
15
32
|
Provides-Extra: docs
|
|
33
|
+
Requires-Dist: sphinx; extra == "docs"
|
|
34
|
+
Requires-Dist: sphinx-astropy; extra == "docs"
|
|
35
|
+
Requires-Dist: sphinx-changelog; extra == "docs"
|
|
36
|
+
Requires-Dist: sphinx-autoapi; extra == "docs"
|
|
37
|
+
Requires-Dist: pytest; extra == "docs"
|
|
38
|
+
Requires-Dist: towncrier; extra == "docs"
|
|
39
|
+
Requires-Dist: dkist-sphinx-theme; extra == "docs"
|
|
40
|
+
Requires-Dist: matplotlib; extra == "docs"
|
|
41
|
+
Requires-Dist: sunpy; extra == "docs"
|
|
16
42
|
|
|
17
43
|
dkist-processing-core
|
|
18
44
|
=====================
|
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
.gitignore
|
|
2
2
|
.pre-commit-config.yaml
|
|
3
3
|
.readthedocs.yml
|
|
4
|
+
.snyk
|
|
4
5
|
CHANGELOG.rst
|
|
5
6
|
README.rst
|
|
6
7
|
bitbucket-pipelines.yml
|
|
7
|
-
check_changelog_updated.sh
|
|
8
8
|
pyproject.toml
|
|
9
|
-
setup.cfg
|
|
10
|
-
setup.py
|
|
11
9
|
changelog/.gitempty
|
|
12
10
|
dkist_processing_core/__init__.py
|
|
13
11
|
dkist_processing_core/build_utils.py
|
|
@@ -46,6 +44,7 @@ docs/automated-processing-deployed.png
|
|
|
46
44
|
docs/changelog.rst
|
|
47
45
|
docs/conf.py
|
|
48
46
|
docs/index.rst
|
|
47
|
+
docs/landing_page.rst
|
|
49
48
|
docs/make.bat
|
|
50
49
|
docs/requirements.txt
|
|
51
50
|
licenses/LICENSE.rst
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
Overview
|
|
2
|
+
========
|
|
3
|
+
|
|
4
|
+
The `dkist-processing-core` code repository provides an abstraction layer between the DKIST data
|
|
5
|
+
processing code, the workflow engine that supports it, and the observability infrastructure of the Data
|
|
6
|
+
Center.
|
|
7
|
+
|
|
8
|
+
.. image:: https://bitbucket.org/dkistdc/dkist-processing-core/raw/faf0c57f2155d03889fcd54bc1676a8a219f6ee3/docs/auto_proc_brick.png
|
|
9
|
+
:width: 600
|
|
10
|
+
:alt: Core, Common, and Instrument Brick Diagram
|
|
11
|
+
|
|
12
|
+
|
|
|
13
|
+
|
|
14
|
+
Four main entities implement the abstraction:
|
|
15
|
+
|
|
16
|
+
*Task* : The Task defines the interface used by a processing pipeline for a step in a workflow.
|
|
17
|
+
By conforming to this interface (i.e. subclassing), the processing pipelines can remain agnostic
|
|
18
|
+
of how the tasks will ultimately be run. The Task additionally implements some methods that should
|
|
19
|
+
be global for all DKIST processing tasks based on the infrastructure it will run on (e.g.
|
|
20
|
+
application performance monitoring infrastructure).
|
|
21
|
+
|
|
22
|
+
*Node* : The job of the Node is to translate a Task into code that can instantiate that task.
|
|
23
|
+
Instantiations of a Task can vary depending on the target environment, e.g. a virtual environment
|
|
24
|
+
with a BashOperator for the workflow engine vs. straight python for a notebook.
|
|
25
|
+
|
|
26
|
+
*Workflow* : The Workflow defines the interface used by the processing pipeline to chain tasks
|
|
27
|
+
together in a directed acyclic graph (DAG). The Workflow transforms this graph into the workflow
|
|
28
|
+
engine format by providing any wrapping boilerplate, task ordering, and selecting the appropriate
|
|
29
|
+
Node instantiation.
|
|
30
|
+
|
|
31
|
+
*Build Utils* : The Build Utils are the capstone layer which eases the transformation process
|
|
32
|
+
for multiple workflows at a time during a processing pipeline's build process.
|
|
33
|
+
|
|
34
|
+
The Workflow and Task are the primary objects used by client libraries.
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=64", "setuptools_scm>=8"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[tool.setuptools.packages]
|
|
6
|
+
find = {}
|
|
7
|
+
|
|
8
|
+
[tool.setuptools_scm]
|
|
9
|
+
|
|
10
|
+
[project]
|
|
11
|
+
dynamic = ["version"]
|
|
12
|
+
name = "dkist-processing-core"
|
|
13
|
+
requires-python = ">=3.11"
|
|
14
|
+
description = "Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow"
|
|
15
|
+
readme = "README.rst"
|
|
16
|
+
authors = [ {name = "NSO / AURA", email = "dkistdc@nso.edu"} ]
|
|
17
|
+
license = {text = "BSD-3-Clause"}
|
|
18
|
+
classifiers = [
|
|
19
|
+
"Programming Language :: Python",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
]
|
|
23
|
+
dependencies = [
|
|
24
|
+
"apache-airflow[postgres, celery] == 2.10.4",
|
|
25
|
+
"elastic-apm < 7.0.0",
|
|
26
|
+
"requests >= 2.23",
|
|
27
|
+
"talus >= 1.1.0, <2.0",
|
|
28
|
+
"pendulum",
|
|
29
|
+
"nbformat >= 5.9.2",
|
|
30
|
+
"dkist-service-configuration >=2.0.2, <3.0",
|
|
31
|
+
"pydantic > 2.0",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[project.urls]
|
|
35
|
+
Homepage = "https://nso.edu/dkist/data-center/"
|
|
36
|
+
Repository = "https://bitbucket.org/dkistdc/dkist-processing-core/"
|
|
37
|
+
Documentation = "https://docs.dkist.nso.edu/projects/core"
|
|
38
|
+
Help = "https://nso.atlassian.net/servicedesk/customer/portal/5"
|
|
39
|
+
|
|
40
|
+
[project.optional-dependencies]
|
|
41
|
+
test = [
|
|
42
|
+
"pytest",
|
|
43
|
+
"pytest-cov",
|
|
44
|
+
"pytest-mock",
|
|
45
|
+
"jinja2",
|
|
46
|
+
"towncrier",
|
|
47
|
+
"nbconvert",
|
|
48
|
+
"ipython",
|
|
49
|
+
]
|
|
50
|
+
docs = [
|
|
51
|
+
"sphinx",
|
|
52
|
+
"sphinx-astropy",
|
|
53
|
+
"sphinx-changelog",
|
|
54
|
+
"sphinx-autoapi",
|
|
55
|
+
"pytest",
|
|
56
|
+
"towncrier",
|
|
57
|
+
"dkist-sphinx-theme",
|
|
58
|
+
"matplotlib",
|
|
59
|
+
"sunpy",
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
[tool.pytest.ini_options]
|
|
63
|
+
markers = [
|
|
64
|
+
"development: For tests that can only be run while developing with a sidecar proxy (as opposed to in bitbucket pipelines)",
|
|
65
|
+
"long: For tests that take a long time to run",
|
|
66
|
+
]
|
|
67
|
+
|
|
68
|
+
[tool.coverage.run]
|
|
69
|
+
omit = [
|
|
70
|
+
"dkist_processing_core/tests/*",
|
|
71
|
+
"dkist_processing_core/__init__.py",
|
|
72
|
+
]
|
|
73
|
+
|
|
74
|
+
[tool.coverage.report]
|
|
75
|
+
show_missing = true
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
[tool.towncrier]
|
|
79
|
+
package = "dkist_processing_core"
|
|
80
|
+
filename = "CHANGELOG.rst"
|
|
81
|
+
directory = "changelog/"
|
|
82
|
+
issue_format = "`#{issue} <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/{issue}>`__"
|
|
83
|
+
title_format = "{version} ({project_date})"
|
|
84
|
+
ignore = [".gitempty"]
|
|
85
|
+
|
|
86
|
+
[[tool.towncrier.type]]
|
|
87
|
+
directory = "feature"
|
|
88
|
+
name = "Features"
|
|
89
|
+
showcontent = true
|
|
90
|
+
|
|
91
|
+
[[tool.towncrier.type]]
|
|
92
|
+
directory = "bugfix"
|
|
93
|
+
name = "Bugfixes"
|
|
94
|
+
showcontent = true
|
|
95
|
+
|
|
96
|
+
[[tool.towncrier.type]]
|
|
97
|
+
directory = "removal"
|
|
98
|
+
name = "Removals"
|
|
99
|
+
showcontent = true
|
|
100
|
+
|
|
101
|
+
[[tool.towncrier.type]]
|
|
102
|
+
directory = "misc"
|
|
103
|
+
name = "Misc"
|
|
104
|
+
showcontent = true
|
|
105
|
+
|
|
106
|
+
[[tool.towncrier.type]]
|
|
107
|
+
directory = "doc"
|
|
108
|
+
name = "Documentation"
|
|
109
|
+
showcontent = true
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
#!/bin/bash
|
|
2
|
-
|
|
3
|
-
CURRENT_VERSION=`git describe --tags --match v* --exact-match`
|
|
4
|
-
ERROR_STR="CHANGELOG has not been updated for ${CURRENT_VERSION}\\nSee the README for instructions"
|
|
5
|
-
ERROR_PAD="***********"
|
|
6
|
-
|
|
7
|
-
if [[ $CURRENT_VERSION =~ "rc" ]]; then
|
|
8
|
-
echo "release candidate detected"
|
|
9
|
-
exit 0;
|
|
10
|
-
fi
|
|
11
|
-
|
|
12
|
-
head -n 1 CHANGELOG.rst | grep -q $CURRENT_VERSION - || {
|
|
13
|
-
echo -e "\\n${ERROR_PAD}"
|
|
14
|
-
echo -e $ERROR_STR;
|
|
15
|
-
echo -e "${ERROR_PAD}"
|
|
16
|
-
exit 1; }
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
dkist_processing_core
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
[tool.towncrier]
|
|
2
|
-
package = "dkist_processing_core"
|
|
3
|
-
filename = "CHANGELOG.rst"
|
|
4
|
-
directory = "changelog/"
|
|
5
|
-
issue_format = "`#{issue} <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/{issue}>`__"
|
|
6
|
-
title_format = "{version} ({project_date})"
|
|
7
|
-
ignore = [".gitempty"]
|
|
8
|
-
|
|
9
|
-
[[tool.towncrier.type]]
|
|
10
|
-
directory = "feature"
|
|
11
|
-
name = "Features"
|
|
12
|
-
showcontent = true
|
|
13
|
-
|
|
14
|
-
[[tool.towncrier.type]]
|
|
15
|
-
directory = "bugfix"
|
|
16
|
-
name = "Bugfixes"
|
|
17
|
-
showcontent = true
|
|
18
|
-
|
|
19
|
-
[[tool.towncrier.type]]
|
|
20
|
-
directory = "removal"
|
|
21
|
-
name = "Removals"
|
|
22
|
-
showcontent = true
|
|
23
|
-
|
|
24
|
-
[[tool.towncrier.type]]
|
|
25
|
-
directory = "misc"
|
|
26
|
-
name = "Misc"
|
|
27
|
-
showcontent = true
|
|
28
|
-
|
|
29
|
-
[[tool.towncrier.type]]
|
|
30
|
-
directory = "doc"
|
|
31
|
-
name = "Documentation"
|
|
32
|
-
showcontent = true
|
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
[metadata]
|
|
2
|
-
name = dkist-processing-core
|
|
3
|
-
description = Abstraction layer that is used by the DKIST Science Data Processing pipelines to process DKIST data using Apache Airflow.
|
|
4
|
-
long_description = file: README.rst
|
|
5
|
-
author = NSO / AURA
|
|
6
|
-
author_email = "dkistdc@nso.edu"
|
|
7
|
-
license = MIT
|
|
8
|
-
url = https://bitbucket.org/dkistdc/dkist-processing-core/src/main/
|
|
9
|
-
project_urls =
|
|
10
|
-
Documentation = https://docs.dkist.nso.edu/projects/core
|
|
11
|
-
classifiers =
|
|
12
|
-
Programming Language :: Python
|
|
13
|
-
Programming Language :: Python :: 3
|
|
14
|
-
Programming Language :: Python :: 3.11
|
|
15
|
-
|
|
16
|
-
[options]
|
|
17
|
-
python_requires = >=3.11
|
|
18
|
-
setup_requires = setuptools_scm
|
|
19
|
-
packages = find:
|
|
20
|
-
include_package_data = True
|
|
21
|
-
install_requires =
|
|
22
|
-
apache-airflow[postgres, celery] == 2.10.2
|
|
23
|
-
elastic-apm < 7.0.0
|
|
24
|
-
requests >= 2.23
|
|
25
|
-
talus >= 1.1.0, <2.0
|
|
26
|
-
pendulum
|
|
27
|
-
nbformat >= 5.9.2
|
|
28
|
-
dkist-service-configuration >=2.0.2, <3.0
|
|
29
|
-
pydantic > 2.0
|
|
30
|
-
|
|
31
|
-
[options.extras_require]
|
|
32
|
-
test =
|
|
33
|
-
pytest
|
|
34
|
-
pytest-cov
|
|
35
|
-
pytest-mock
|
|
36
|
-
jinja2
|
|
37
|
-
towncrier
|
|
38
|
-
nbconvert
|
|
39
|
-
ipython
|
|
40
|
-
docs =
|
|
41
|
-
sphinx
|
|
42
|
-
sphinx-astropy
|
|
43
|
-
sphinx-changelog
|
|
44
|
-
sphinx-autoapi
|
|
45
|
-
pytest
|
|
46
|
-
towncrier
|
|
47
|
-
dkist-sphinx-theme
|
|
48
|
-
matplotlib
|
|
49
|
-
sunpy
|
|
50
|
-
|
|
51
|
-
[build_docs]
|
|
52
|
-
source-dir = docs
|
|
53
|
-
build-dir = docs/_build
|
|
54
|
-
all_files = 1
|
|
55
|
-
|
|
56
|
-
[upload_docs]
|
|
57
|
-
upload-dir = docs/_build/html
|
|
58
|
-
show-response = 1
|
|
59
|
-
|
|
60
|
-
[tool:pytest]
|
|
61
|
-
markers =
|
|
62
|
-
development: For tests that can only be run while developing with a sidecar proxy (as opposed to in bitbucket pipelines)
|
|
63
|
-
long: For tests that take a long time to run.
|
|
64
|
-
|
|
65
|
-
[coverage:run]
|
|
66
|
-
omit =
|
|
67
|
-
dkist_processing_core/tests/*
|
|
68
|
-
dkist_processing_core/__init__.py
|
|
69
|
-
|
|
70
|
-
[egg_info]
|
|
71
|
-
tag_build =
|
|
72
|
-
tag_date = 0
|
|
73
|
-
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/__init__.py
RENAMED
|
File without changes
|
{dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/build_utils.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/resource_queue.py
RENAMED
|
File without changes
|
|
File without changes
|
{dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/__init__.py
RENAMED
|
File without changes
|
{dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/conftest.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/test_task.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/auto-proc-concept-model.png
RENAMED
|
File without changes
|
|
File without changes
|
{dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/automated-processing-deployed.png
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|