dkist-processing-core 4.2.1__tar.gz → 5.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. dkist_processing_core-5.0.0/.snyk +3 -0
  2. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/CHANGELOG.rst +35 -0
  3. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/PKG-INFO +33 -7
  4. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/bitbucket-pipelines.yml +25 -30
  5. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/node.py +1 -1
  6. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/test_node.py +22 -3
  7. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/workflow.py +1 -1
  8. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core.egg-info/PKG-INFO +33 -7
  9. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core.egg-info/SOURCES.txt +2 -3
  10. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core.egg-info/requires.txt +1 -1
  11. dkist_processing_core-5.0.0/dkist_processing_core.egg-info/top_level.txt +6 -0
  12. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/index.rst +1 -1
  13. dkist_processing_core-5.0.0/docs/landing_page.rst +34 -0
  14. dkist_processing_core-5.0.0/pyproject.toml +109 -0
  15. dkist_processing_core-5.0.0/setup.cfg +4 -0
  16. dkist-processing-core-4.2.1/check_changelog_updated.sh +0 -16
  17. dkist-processing-core-4.2.1/dkist_processing_core.egg-info/top_level.txt +0 -1
  18. dkist-processing-core-4.2.1/pyproject.toml +0 -32
  19. dkist-processing-core-4.2.1/setup.cfg +0 -73
  20. dkist-processing-core-4.2.1/setup.py +0 -4
  21. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/.gitignore +0 -0
  22. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/.pre-commit-config.yaml +0 -0
  23. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/.readthedocs.yml +0 -0
  24. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/README.rst +0 -0
  25. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/changelog/.gitempty +0 -0
  26. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/__init__.py +0 -0
  27. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/build_utils.py +0 -0
  28. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/config.py +0 -0
  29. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/failure_callback.py +0 -0
  30. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/resource_queue.py +0 -0
  31. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/task.py +0 -0
  32. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/__init__.py +0 -0
  33. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/conftest.py +0 -0
  34. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/invalid_workflow_cyclic/__init__.py +0 -0
  35. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/invalid_workflow_cyclic/workflow.py +0 -0
  36. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/invalid_workflow_for_docker_multi_category/__init__.py +0 -0
  37. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/invalid_workflow_for_docker_multi_category/workflow.py +0 -0
  38. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/task_example.py +0 -0
  39. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/test_build_utils.py +0 -0
  40. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/test_export.py +0 -0
  41. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/test_failure_callback.py +0 -0
  42. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/test_task.py +0 -0
  43. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/test_workflow.py +0 -0
  44. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/valid_workflow_package/__init__.py +0 -0
  45. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/valid_workflow_package/workflow.py +0 -0
  46. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/zero_node_workflow_package/__init__.py +0 -0
  47. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core/tests/zero_node_workflow_package/workflow.py +0 -0
  48. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/dkist_processing_core.egg-info/dependency_links.txt +0 -0
  49. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/Makefile +0 -0
  50. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/auto-proc-concept-model.png +0 -0
  51. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/auto_proc_brick.png +0 -0
  52. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/automated-processing-deployed.png +0 -0
  53. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/changelog.rst +0 -0
  54. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/conf.py +0 -0
  55. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/make.bat +0 -0
  56. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/docs/requirements.txt +0 -0
  57. {dkist-processing-core-4.2.1 → dkist_processing_core-5.0.0}/licenses/LICENSE.rst +0 -0
@@ -0,0 +1,3 @@
1
+ # Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities
2
+ # https://docs.snyk.io/manage-risk/policies/the-.snyk-file#syntax-of-the-.snyk-file
3
+ version: v1.25.0
@@ -1,3 +1,38 @@
1
+ v5.0.0 (2025-02-14)
2
+ ===================
3
+
4
+ Features
5
+ --------
6
+
7
+ - Instrument python packages are now installed using their "frozen" pip extra, which contains a complete set of frozen dependencies.
8
+ This ensures the environment is exactly the same for a given version of an instrument repo. (`#48 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/48>`__)
9
+
10
+
11
+ Misc
12
+ ----
13
+
14
+ - Update bitbucket pipelines to use common scripts for checking for changelog snippets and verifying doc builds. (`#47 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/47>`__)
15
+ - Update Bitbucket pipelines to use execute script for standard steps. (`#49 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/49>`__)
16
+
17
+
18
+ v4.3.0 (2025-01-08)
19
+ ===================
20
+
21
+ Misc
22
+ ----
23
+
24
+ - Make and publish wheels at code push in build pipeline (`#43 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/43>`__)
25
+ - Switch from setup.cfg to pyproject.toml for build configuration (`#43 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/43>`__)
26
+ - Update Bitbucket pipelines to use standardized lint and scan steps. (`#45 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/45>`__)
27
+ - Upgrade apache-airflow to 2.10.4. (`#46 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/46>`__)
28
+
29
+
30
+ Documentation
31
+ -------------
32
+
33
+ - Change the documentation landing page to focus more on users and less on developers. (`#44 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/44>`__)
34
+
35
+
1
36
  v4.2.1 (2024-09-27)
2
37
  ===================
3
38
 
@@ -1,18 +1,44 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.2
2
2
  Name: dkist-processing-core
3
- Version: 4.2.1
4
- Summary: Abstraction layer that is used by the DKIST Science Data Processing pipelines to process DKIST data using Apache Airflow.
5
- Home-page: https://bitbucket.org/dkistdc/dkist-processing-core/src/main/
6
- Author: NSO / AURA
7
- Author-email: "dkistdc@nso.edu"
8
- License: MIT
3
+ Version: 5.0.0
4
+ Summary: Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow
5
+ Author-email: NSO / AURA <dkistdc@nso.edu>
6
+ License: BSD-3-Clause
7
+ Project-URL: Homepage, https://nso.edu/dkist/data-center/
8
+ Project-URL: Repository, https://bitbucket.org/dkistdc/dkist-processing-core/
9
9
  Project-URL: Documentation, https://docs.dkist.nso.edu/projects/core
10
+ Project-URL: Help, https://nso.atlassian.net/servicedesk/customer/portal/5
10
11
  Classifier: Programming Language :: Python
11
12
  Classifier: Programming Language :: Python :: 3
12
13
  Classifier: Programming Language :: Python :: 3.11
13
14
  Requires-Python: >=3.11
15
+ Description-Content-Type: text/x-rst
16
+ Requires-Dist: apache-airflow[celery,postgres]==2.10.4
17
+ Requires-Dist: elastic-apm<7.0.0
18
+ Requires-Dist: requests>=2.23
19
+ Requires-Dist: talus<2.0,>=1.1.0
20
+ Requires-Dist: pendulum
21
+ Requires-Dist: nbformat>=5.9.2
22
+ Requires-Dist: dkist-service-configuration<3.0,>=2.0.2
23
+ Requires-Dist: pydantic>2.0
14
24
  Provides-Extra: test
25
+ Requires-Dist: pytest; extra == "test"
26
+ Requires-Dist: pytest-cov; extra == "test"
27
+ Requires-Dist: pytest-mock; extra == "test"
28
+ Requires-Dist: jinja2; extra == "test"
29
+ Requires-Dist: towncrier; extra == "test"
30
+ Requires-Dist: nbconvert; extra == "test"
31
+ Requires-Dist: ipython; extra == "test"
15
32
  Provides-Extra: docs
33
+ Requires-Dist: sphinx; extra == "docs"
34
+ Requires-Dist: sphinx-astropy; extra == "docs"
35
+ Requires-Dist: sphinx-changelog; extra == "docs"
36
+ Requires-Dist: sphinx-autoapi; extra == "docs"
37
+ Requires-Dist: pytest; extra == "docs"
38
+ Requires-Dist: towncrier; extra == "docs"
39
+ Requires-Dist: dkist-sphinx-theme; extra == "docs"
40
+ Requires-Dist: matplotlib; extra == "docs"
41
+ Requires-Dist: sunpy; extra == "docs"
16
42
 
17
43
  dkist-processing-core
18
44
  =====================
@@ -6,41 +6,34 @@ definitions:
6
6
  ports:
7
7
  - 5672:5672
8
8
  steps:
9
+ - step: &download_execute_script
10
+ name: Download Script Execute Script
11
+ script:
12
+ - echo "Retrieving Script Execution Script"
13
+ - curl -fL https://getcli.jfrog.io | sh
14
+ - ./jfrog rt dl --url $ARTIFACTORY_URL --user $ARTIFACTORY_USER --password $ARTIFACTORY_PASSWORD generic-packages/ci_scripts/latest/execute_script.sh
15
+ - cat ci_scripts/latest/execute_script.sh
16
+ - chmod 755 ci_scripts/latest/execute_script.sh
17
+ artifacts:
18
+ - ci_scripts/latest/execute_script.sh
9
19
  - step: &lint
10
20
  caches:
11
21
  - pip
12
22
  name: Lint
13
23
  script:
14
- - pip install pre-commit
15
- - pre-commit install
16
- - pre-commit run --all-files
24
+ - ./ci_scripts/latest/execute_script.sh lint_python.sh
17
25
  - step: &changelog
18
26
  caches:
19
27
  - pip
20
28
  name: Changelog
21
29
  script:
22
- - pip install towncrier
23
- # We need to fetch the main branch to compare against
24
- - git remote set-branches --add origin main
25
- - git fetch origin main
26
- - towncrier check --compare-with origin/main
30
+ - ./ci_scripts/latest/execute_script.sh check_changelog.sh
27
31
  - step: &scan
28
32
  caches:
29
33
  - pip
30
34
  name: Scan
31
35
  script:
32
- - pip install .
33
- - pip freeze | grep -v @ > requirements.txt
34
- - cat requirements.txt
35
- - echo $SNYK_VERSION
36
- - curl -L -o snyk https://github.com/snyk/snyk/releases/download/$SNYK_VERSION/snyk-linux
37
- - chmod 755 snyk
38
- - ./snyk -d auth $SNYK_TOKEN
39
- - echo $SNYK_IGNORE
40
- - for id in $SNYK_IGNORE; do echo Ignoring $id; ./snyk ignore $id; done
41
- - cat .snyk || echo "No .snyk found. Probably because there was nothing to ignore."
42
- - echo $SNYK_CLI_COMMAND
43
- - $SNYK_CLI_COMMAND
36
+ - ./ci_scripts/latest/execute_script.sh scan_python.sh
44
37
  - step: &test
45
38
  caches:
46
39
  - pip
@@ -52,19 +45,12 @@ definitions:
52
45
  script:
53
46
  - pip install .[test]
54
47
  - pytest -m "not development" --cov dkist_processing_core
55
-
56
48
  - step: &push
57
49
  caches:
58
50
  - pip
59
51
  name: Push
60
52
  script:
61
- - python setup.py sdist
62
- - pip install twine
63
- - twine upload dist/*
64
- - step: &check_changelog
65
- name: Check for updated CHANGELOG
66
- script:
67
- - ./check_changelog_updated.sh
53
+ - ./ci_scripts/latest/execute_script.sh push_python.sh
68
54
  - step: &build_mpw
69
55
  caches:
70
56
  - pip
@@ -79,24 +65,33 @@ definitions:
79
65
  - cat Dockerfile
80
66
  - docker login dkistdc-docker.jfrog.io --username $DOCKER_USERNAME --password $DOCKER_PASSWORD
81
67
  - docker build -t $DOCKER_HUB_URL/${BITBUCKET_REPO_SLUG}:latest .
68
+ - step: &docs
69
+ name: Test Docs
70
+ caches:
71
+ - pip
72
+ script:
73
+ - ./ci_scripts/latest/execute_script.sh test_sphinx.sh
82
74
 
83
75
  pipelines:
84
76
  default:
77
+ - step: *download_execute_script
85
78
  - parallel:
86
79
  - step: *lint
87
80
  - step: *changelog
88
81
  - parallel:
89
82
  - step: *scan
90
83
  - step: *test
84
+ - step: *docs
91
85
  - step: *build_mpw
92
-
93
86
  tags:
94
87
  'v*':
88
+ - step: *download_execute_script
95
89
  - parallel:
96
- - step: *check_changelog
90
+ - step: *changelog
97
91
  - step: *lint
98
92
  - parallel:
99
93
  - step: *scan
100
94
  - step: *test
95
+ - step: *docs
101
96
  - step: *build_mpw
102
97
  - step: *push
@@ -44,7 +44,7 @@ class Node:
44
44
  self.workflow_package = workflow_package
45
45
  self.upstreams = upstreams
46
46
  self.resource_queue = resource_queue
47
- self.pip_extras = pip_extras
47
+ self.pip_extras = ["frozen"] + (pip_extras or [])
48
48
 
49
49
  @property
50
50
  def operator(self) -> BashOperator:
@@ -13,11 +13,16 @@ from dkist_processing_core.tests.task_example import Task
13
13
 
14
14
 
15
15
  @pytest.fixture()
16
- def single_node(pip_extras) -> Node:
16
+ def single_node_version() -> str:
17
+ return "3.4.5"
18
+
19
+
20
+ @pytest.fixture()
21
+ def single_node(pip_extras, single_node_version) -> Node:
17
22
  """A single Node instance."""
18
23
  return Node(
19
24
  workflow_name="test_node",
20
- workflow_version="v1",
25
+ workflow_version=single_node_version,
21
26
  workflow_package=__package__,
22
27
  task=Task,
23
28
  resource_queue=ResourceQueue.DEFAULT,
@@ -45,7 +50,21 @@ def test_nodes(node, fake_producer_factory, queue_name, pip_extras):
45
50
  assert node.task == task
46
51
  assert node.workflow_version == version
47
52
  assert node.resource_queue == queue_name
48
- assert node.pip_extras == pip_extras
53
+ assert node.pip_extras == ["frozen"] + (pip_extras or [])
54
+
55
+
56
+ def test_node_install_command(single_node, pip_extras, single_node_version):
57
+ """
58
+ Given: A valid node instance.
59
+ When: Creating the pip install commands with and without pip extras
60
+ Then: The correct command is generated; pip extras are installed separately from the "frozen" extra
61
+ """
62
+ expected_repo_name = __package__.split(".")[0].replace("_", "-")
63
+ extras_str = f"'[{','.join(['frozen'] + (pip_extras or []))}]'"
64
+ expected_install_command = f"""python -m pip install --upgrade pip
65
+ python -m pip install {expected_repo_name}{extras_str}=={single_node_version}"""
66
+
67
+ assert single_node.install_command == expected_install_command
49
68
 
50
69
 
51
70
  @pytest.mark.long()
@@ -7,11 +7,11 @@ from pathlib import Path
7
7
  import nbformat as nbf
8
8
  from airflow import DAG
9
9
 
10
- from dkist_processing_core import ResourceQueue
11
10
  from dkist_processing_core.config import core_configurations
12
11
  from dkist_processing_core.node import Node
13
12
  from dkist_processing_core.node import task_type_hint
14
13
  from dkist_processing_core.node import upstreams_type_hint
14
+ from dkist_processing_core.resource_queue import ResourceQueue
15
15
 
16
16
  __all__ = ["Workflow", "workflow_name_from_details"]
17
17
 
@@ -1,18 +1,44 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.2
2
2
  Name: dkist-processing-core
3
- Version: 4.2.1
4
- Summary: Abstraction layer that is used by the DKIST Science Data Processing pipelines to process DKIST data using Apache Airflow.
5
- Home-page: https://bitbucket.org/dkistdc/dkist-processing-core/src/main/
6
- Author: NSO / AURA
7
- Author-email: "dkistdc@nso.edu"
8
- License: MIT
3
+ Version: 5.0.0
4
+ Summary: Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow
5
+ Author-email: NSO / AURA <dkistdc@nso.edu>
6
+ License: BSD-3-Clause
7
+ Project-URL: Homepage, https://nso.edu/dkist/data-center/
8
+ Project-URL: Repository, https://bitbucket.org/dkistdc/dkist-processing-core/
9
9
  Project-URL: Documentation, https://docs.dkist.nso.edu/projects/core
10
+ Project-URL: Help, https://nso.atlassian.net/servicedesk/customer/portal/5
10
11
  Classifier: Programming Language :: Python
11
12
  Classifier: Programming Language :: Python :: 3
12
13
  Classifier: Programming Language :: Python :: 3.11
13
14
  Requires-Python: >=3.11
15
+ Description-Content-Type: text/x-rst
16
+ Requires-Dist: apache-airflow[celery,postgres]==2.10.4
17
+ Requires-Dist: elastic-apm<7.0.0
18
+ Requires-Dist: requests>=2.23
19
+ Requires-Dist: talus<2.0,>=1.1.0
20
+ Requires-Dist: pendulum
21
+ Requires-Dist: nbformat>=5.9.2
22
+ Requires-Dist: dkist-service-configuration<3.0,>=2.0.2
23
+ Requires-Dist: pydantic>2.0
14
24
  Provides-Extra: test
25
+ Requires-Dist: pytest; extra == "test"
26
+ Requires-Dist: pytest-cov; extra == "test"
27
+ Requires-Dist: pytest-mock; extra == "test"
28
+ Requires-Dist: jinja2; extra == "test"
29
+ Requires-Dist: towncrier; extra == "test"
30
+ Requires-Dist: nbconvert; extra == "test"
31
+ Requires-Dist: ipython; extra == "test"
15
32
  Provides-Extra: docs
33
+ Requires-Dist: sphinx; extra == "docs"
34
+ Requires-Dist: sphinx-astropy; extra == "docs"
35
+ Requires-Dist: sphinx-changelog; extra == "docs"
36
+ Requires-Dist: sphinx-autoapi; extra == "docs"
37
+ Requires-Dist: pytest; extra == "docs"
38
+ Requires-Dist: towncrier; extra == "docs"
39
+ Requires-Dist: dkist-sphinx-theme; extra == "docs"
40
+ Requires-Dist: matplotlib; extra == "docs"
41
+ Requires-Dist: sunpy; extra == "docs"
16
42
 
17
43
  dkist-processing-core
18
44
  =====================
@@ -1,13 +1,11 @@
1
1
  .gitignore
2
2
  .pre-commit-config.yaml
3
3
  .readthedocs.yml
4
+ .snyk
4
5
  CHANGELOG.rst
5
6
  README.rst
6
7
  bitbucket-pipelines.yml
7
- check_changelog_updated.sh
8
8
  pyproject.toml
9
- setup.cfg
10
- setup.py
11
9
  changelog/.gitempty
12
10
  dkist_processing_core/__init__.py
13
11
  dkist_processing_core/build_utils.py
@@ -46,6 +44,7 @@ docs/automated-processing-deployed.png
46
44
  docs/changelog.rst
47
45
  docs/conf.py
48
46
  docs/index.rst
47
+ docs/landing_page.rst
49
48
  docs/make.bat
50
49
  docs/requirements.txt
51
50
  licenses/LICENSE.rst
@@ -1,4 +1,4 @@
1
- apache-airflow[celery,postgres]==2.10.2
1
+ apache-airflow[celery,postgres]==2.10.4
2
2
  elastic-apm<7.0.0
3
3
  requests>=2.23
4
4
  talus<2.0,>=1.1.0
@@ -0,0 +1,6 @@
1
+ changelog
2
+ ci_scripts
3
+ dkist_processing_core
4
+ docs
5
+ licenses
6
+ wheelhouse
@@ -1,4 +1,4 @@
1
- .. include:: ../README.rst
1
+ .. include:: landing_page.rst
2
2
 
3
3
  .. toctree::
4
4
  :maxdepth: 2
@@ -0,0 +1,34 @@
1
+ Overview
2
+ ========
3
+
4
+ The `dkist-processing-core` code repository provides an abstraction layer between the DKIST data
5
+ processing code, the workflow engine that supports it, and the observability infrastructure of the Data
6
+ Center.
7
+
8
+ .. image:: https://bitbucket.org/dkistdc/dkist-processing-core/raw/faf0c57f2155d03889fcd54bc1676a8a219f6ee3/docs/auto_proc_brick.png
9
+ :width: 600
10
+ :alt: Core, Common, and Instrument Brick Diagram
11
+
12
+ |
13
+
14
+ Four main entities implement the abstraction:
15
+
16
+ *Task* : The Task defines the interface used by a processing pipeline for a step in a workflow.
17
+ By conforming to this interface (i.e. subclassing), the processing pipelines can remain agnostic
18
+ of how the tasks will ultimately be run. The Task additionally implements some methods that should
19
+ be global for all DKIST processing tasks based on the infrastructure it will run on (e.g.
20
+ application performance monitoring infrastructure).
21
+
22
+ *Node* : The job of the Node is to translate a Task into code that can instantiate that task.
23
+ Instantiations of a Task can vary depending on the target environment, e.g. a virtual environment
24
+ with a BashOperator for the workflow engine vs. straight python for a notebook.
25
+
26
+ *Workflow* : The Workflow defines the interface used by the processing pipeline to chain tasks
27
+ together in a directed acyclic graph (DAG). The Workflow transforms this graph into the workflow
28
+ engine format by providing any wrapping boilerplate, task ordering, and selecting the appropriate
29
+ Node instantiation.
30
+
31
+ *Build Utils* : The Build Utils are the capstone layer which eases the transformation process
32
+ for multiple workflows at a time during a processing pipeline's build process.
33
+
34
+ The Workflow and Task are the primary objects used by client libraries.
@@ -0,0 +1,109 @@
1
+ [build-system]
2
+ requires = ["setuptools>=64", "setuptools_scm>=8"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [tool.setuptools.packages]
6
+ find = {}
7
+
8
+ [tool.setuptools_scm]
9
+
10
+ [project]
11
+ dynamic = ["version"]
12
+ name = "dkist-processing-core"
13
+ requires-python = ">=3.11"
14
+ description = "Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow"
15
+ readme = "README.rst"
16
+ authors = [ {name = "NSO / AURA", email = "dkistdc@nso.edu"} ]
17
+ license = {text = "BSD-3-Clause"}
18
+ classifiers = [
19
+ "Programming Language :: Python",
20
+ "Programming Language :: Python :: 3",
21
+ "Programming Language :: Python :: 3.11",
22
+ ]
23
+ dependencies = [
24
+ "apache-airflow[postgres, celery] == 2.10.4",
25
+ "elastic-apm < 7.0.0",
26
+ "requests >= 2.23",
27
+ "talus >= 1.1.0, <2.0",
28
+ "pendulum",
29
+ "nbformat >= 5.9.2",
30
+ "dkist-service-configuration >=2.0.2, <3.0",
31
+ "pydantic > 2.0",
32
+ ]
33
+
34
+ [project.urls]
35
+ Homepage = "https://nso.edu/dkist/data-center/"
36
+ Repository = "https://bitbucket.org/dkistdc/dkist-processing-core/"
37
+ Documentation = "https://docs.dkist.nso.edu/projects/core"
38
+ Help = "https://nso.atlassian.net/servicedesk/customer/portal/5"
39
+
40
+ [project.optional-dependencies]
41
+ test = [
42
+ "pytest",
43
+ "pytest-cov",
44
+ "pytest-mock",
45
+ "jinja2",
46
+ "towncrier",
47
+ "nbconvert",
48
+ "ipython",
49
+ ]
50
+ docs = [
51
+ "sphinx",
52
+ "sphinx-astropy",
53
+ "sphinx-changelog",
54
+ "sphinx-autoapi",
55
+ "pytest",
56
+ "towncrier",
57
+ "dkist-sphinx-theme",
58
+ "matplotlib",
59
+ "sunpy",
60
+ ]
61
+
62
+ [tool.pytest.ini_options]
63
+ markers = [
64
+ "development: For tests that can only be run while developing with a sidecar proxy (as opposed to in bitbucket pipelines)",
65
+ "long: For tests that take a long time to run",
66
+ ]
67
+
68
+ [tool.coverage.run]
69
+ omit = [
70
+ "dkist_processing_core/tests/*",
71
+ "dkist_processing_core/__init__.py",
72
+ ]
73
+
74
+ [tool.coverage.report]
75
+ show_missing = true
76
+
77
+
78
+ [tool.towncrier]
79
+ package = "dkist_processing_core"
80
+ filename = "CHANGELOG.rst"
81
+ directory = "changelog/"
82
+ issue_format = "`#{issue} <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/{issue}>`__"
83
+ title_format = "{version} ({project_date})"
84
+ ignore = [".gitempty"]
85
+
86
+ [[tool.towncrier.type]]
87
+ directory = "feature"
88
+ name = "Features"
89
+ showcontent = true
90
+
91
+ [[tool.towncrier.type]]
92
+ directory = "bugfix"
93
+ name = "Bugfixes"
94
+ showcontent = true
95
+
96
+ [[tool.towncrier.type]]
97
+ directory = "removal"
98
+ name = "Removals"
99
+ showcontent = true
100
+
101
+ [[tool.towncrier.type]]
102
+ directory = "misc"
103
+ name = "Misc"
104
+ showcontent = true
105
+
106
+ [[tool.towncrier.type]]
107
+ directory = "doc"
108
+ name = "Documentation"
109
+ showcontent = true
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -1,16 +0,0 @@
1
- #!/bin/bash
2
-
3
- CURRENT_VERSION=`git describe --tags --match v* --exact-match`
4
- ERROR_STR="CHANGELOG has not been updated for ${CURRENT_VERSION}\\nSee the README for instructions"
5
- ERROR_PAD="***********"
6
-
7
- if [[ $CURRENT_VERSION =~ "rc" ]]; then
8
- echo "release candidate detected"
9
- exit 0;
10
- fi
11
-
12
- head -n 1 CHANGELOG.rst | grep -q $CURRENT_VERSION - || {
13
- echo -e "\\n${ERROR_PAD}"
14
- echo -e $ERROR_STR;
15
- echo -e "${ERROR_PAD}"
16
- exit 1; }
@@ -1 +0,0 @@
1
- dkist_processing_core
@@ -1,32 +0,0 @@
1
- [tool.towncrier]
2
- package = "dkist_processing_core"
3
- filename = "CHANGELOG.rst"
4
- directory = "changelog/"
5
- issue_format = "`#{issue} <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/{issue}>`__"
6
- title_format = "{version} ({project_date})"
7
- ignore = [".gitempty"]
8
-
9
- [[tool.towncrier.type]]
10
- directory = "feature"
11
- name = "Features"
12
- showcontent = true
13
-
14
- [[tool.towncrier.type]]
15
- directory = "bugfix"
16
- name = "Bugfixes"
17
- showcontent = true
18
-
19
- [[tool.towncrier.type]]
20
- directory = "removal"
21
- name = "Removals"
22
- showcontent = true
23
-
24
- [[tool.towncrier.type]]
25
- directory = "misc"
26
- name = "Misc"
27
- showcontent = true
28
-
29
- [[tool.towncrier.type]]
30
- directory = "doc"
31
- name = "Documentation"
32
- showcontent = true
@@ -1,73 +0,0 @@
1
- [metadata]
2
- name = dkist-processing-core
3
- description = Abstraction layer that is used by the DKIST Science Data Processing pipelines to process DKIST data using Apache Airflow.
4
- long_description = file: README.rst
5
- author = NSO / AURA
6
- author_email = "dkistdc@nso.edu"
7
- license = MIT
8
- url = https://bitbucket.org/dkistdc/dkist-processing-core/src/main/
9
- project_urls =
10
- Documentation = https://docs.dkist.nso.edu/projects/core
11
- classifiers =
12
- Programming Language :: Python
13
- Programming Language :: Python :: 3
14
- Programming Language :: Python :: 3.11
15
-
16
- [options]
17
- python_requires = >=3.11
18
- setup_requires = setuptools_scm
19
- packages = find:
20
- include_package_data = True
21
- install_requires =
22
- apache-airflow[postgres, celery] == 2.10.2
23
- elastic-apm < 7.0.0
24
- requests >= 2.23
25
- talus >= 1.1.0, <2.0
26
- pendulum
27
- nbformat >= 5.9.2
28
- dkist-service-configuration >=2.0.2, <3.0
29
- pydantic > 2.0
30
-
31
- [options.extras_require]
32
- test =
33
- pytest
34
- pytest-cov
35
- pytest-mock
36
- jinja2
37
- towncrier
38
- nbconvert
39
- ipython
40
- docs =
41
- sphinx
42
- sphinx-astropy
43
- sphinx-changelog
44
- sphinx-autoapi
45
- pytest
46
- towncrier
47
- dkist-sphinx-theme
48
- matplotlib
49
- sunpy
50
-
51
- [build_docs]
52
- source-dir = docs
53
- build-dir = docs/_build
54
- all_files = 1
55
-
56
- [upload_docs]
57
- upload-dir = docs/_build/html
58
- show-response = 1
59
-
60
- [tool:pytest]
61
- markers =
62
- development: For tests that can only be run while developing with a sidecar proxy (as opposed to in bitbucket pipelines)
63
- long: For tests that take a long time to run.
64
-
65
- [coverage:run]
66
- omit =
67
- dkist_processing_core/tests/*
68
- dkist_processing_core/__init__.py
69
-
70
- [egg_info]
71
- tag_build =
72
- tag_date = 0
73
-
@@ -1,4 +0,0 @@
1
- """Use setup.cfg to configure packaging."""
2
- from setuptools import setup
3
-
4
- setup(use_scm_version=True)