dkist-processing-core 5.1.0__tar.gz → 5.2.0__tar.gz

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/.pre-commit-config.yaml

@@ -2,7 +2,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.4.0
+    rev: v5.0.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
@@ -10,14 +10,14 @@ repos:
     -   id: check-added-large-files
     -   id: debug-statements
 -   repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 25.1.0
     hooks:
     -   id: black
-        args: [ "-l 100" ]
--   repo: https://github.com/asottile/reorder_python_imports
-    rev: v2.3.5
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
     hooks:
-    -   id: reorder-python-imports
+    -   id: isort
+        name: isort (python)
 -   repo: https://github.com/pycqa/pydocstyle
     rev: 6.1.1  # pick a git hash / tag to point to
     hooks:

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/CHANGELOG.rst

@@ -1,3 +1,29 @@
+v5.2.0 (2025-09-08)
+===================
+
+Misc
+----
+
+- Update pre-commit hook versions and replace python-reorder-imports with isort. (`#54 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/54>`__)
+- Update dependencies to airflow 2.11.0 and limit to python >=3.12. (`#55 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/55>`__)
+
+
+v5.1.1 (2025-05-22)
+===================
+
+Bugfixes
+--------
+
+- Don't use `--system-site-packages` in pip install command for task dependencies. (`#53 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/53>`__)
+
+
+Misc
+----
+
+- Add coverage badge to README.rst. (`#51 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/51>`__)
+- Add missing build dependency specifications. (`#52 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/52>`__)
+
+
 v5.1.0 (2025-02-24)
 ===================
 

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: dkist-processing-core
-Version: 5.1.0
+Version: 5.2.0
 Summary: Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -10,10 +10,10 @@ Project-URL: Documentation, https://docs.dkist.nso.edu/projects/core
 Project-URL: Help, https://nso.atlassian.net/servicedesk/customer/portal/5
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.11
-Requires-Python: >=3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.12
 Description-Content-Type: text/x-rst
-Requires-Dist: apache-airflow[celery,postgres]==2.10.5
+Requires-Dist: apache-airflow[celery,postgres]==2.11.0
 Requires-Dist: elastic-apm<7.0.0
 Requires-Dist: requests>=2.23
 Requires-Dist: talus<2.0,>=1.1.0
@@ -43,6 +43,8 @@ Requires-Dist: sunpy; extra == "docs"
 dkist-processing-core
 =====================
 
+|codecov|
+
 Overview
 --------
 The dkist-processing-core package provides an abstraction layer between the dkist data processing code, the workflow
@@ -247,3 +249,6 @@ then the Bitbucket pipeline will fail. To be able to use the same tag you must d
     # Re-tag with the same version
     git tag vWHATEVER.THE.VERSION
     git push --tags origin main
+
+.. |codecov| image:: https://codecov.io/bb/dkistdc/dkist-processing-core/graph/badge.svg?token=SB18SCBJ8Q
+ :target: https://codecov.io/bb/dkistdc/dkist-processing-core

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/README.rst

@@ -1,6 +1,8 @@
 dkist-processing-core
 =====================
 
+|codecov|
+
 Overview
 --------
 The dkist-processing-core package provides an abstraction layer between the dkist data processing code, the workflow
@@ -205,3 +207,6 @@ then the Bitbucket pipeline will fail. To be able to use the same tag you must d
     # Re-tag with the same version
     git tag vWHATEVER.THE.VERSION
     git push --tags origin main
+
+.. |codecov| image:: https://codecov.io/bb/dkistdc/dkist-processing-core/graph/badge.svg?token=SB18SCBJ8Q
+ :target: https://codecov.io/bb/dkistdc/dkist-processing-core

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/bitbucket-pipelines.yml

@@ -1,4 +1,4 @@
-image: python:3.11
+image: python:3.12
 definitions:
   services:
     broker:
@@ -16,24 +16,28 @@ definitions:
           - chmod 755 ci_scripts/latest/execute_script.sh
         artifacts:
           - ci_scripts/latest/execute_script.sh
+
     - step: &lint
         caches:
           - pip
         name: Lint
         script:
           - ./ci_scripts/latest/execute_script.sh lint_python.sh
+
     - step: &changelog
         caches:
           - pip
         name: Changelog
         script:
           - ./ci_scripts/latest/execute_script.sh check_changelog.sh
+
     - step: &scan
         caches:
           - pip
         name: Scan
         script:
           - ./ci_scripts/latest/execute_script.sh scan_python.sh
+
     - step: &test
         caches:
           - pip
@@ -43,8 +47,11 @@ definitions:
           - broker
         name: Test
         script:
+          - pip install -U pip wheel build setuptools_scm setuptools
           - pip install .[test]
-          - pytest -m "not development" --cov dkist_processing_core
+          - pytest -m "not development" --cov dkist_processing_core --cov-report=xml --cov-report=term-missing
+          - ./ci_scripts/latest/execute_script.sh upload_coverage.sh
+
     - step: &push
         caches:
           - pip

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/__init__.py

@@ -1,4 +1,5 @@
 """Package-level setup information."""
+
 from importlib.metadata import PackageNotFoundError
 from importlib.metadata import version
 

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/build_utils.py

@@ -1,4 +1,5 @@
 """Utilities for the build pipeline."""
+
 import importlib
 from pathlib import Path
 from shutil import rmtree
@@ -6,7 +7,6 @@ from types import ModuleType
 
 from dkist_processing_core import Workflow
 
-
 __all__ = ["validate_workflows", "export_dags", "export_notebook_dockerfile", "export_notebooks"]
 
 
@@ -109,7 +109,7 @@ class NotebookDockerfile:
     @property
     def preamble(self) -> list[str]:
         """Dockerfile preamble lines."""
-        return ["FROM python:3.11", "ENV LANG=C.UTF-8"]
+        return ["FROM python:3.12", "ENV LANG=C.UTF-8"]
 
     @property
     def setup(self) -> list[str]:

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/config.py

@@ -1,4 +1,5 @@
 """Environment controlled configurations for dkist_processing_core."""
+
 from dkist_service_configuration import MeshServiceConfigurationBase
 from dkist_service_configuration.settings import MeshService
 from pydantic import Field

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/failure_callback.py

@@ -1,4 +1,5 @@
 """Define the failure callback functionality."""
+
 import logging
 from contextlib import contextmanager
 from typing import Callable

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/node.py

@@ -1,4 +1,5 @@
 """Abstraction layer to construct a workflow node using and airflow operator."""
+
 from collections.abc import Iterable
 from typing import Type
 
@@ -7,7 +8,6 @@ from airflow.operators.bash import BashOperator
 from dkist_processing_core.resource_queue import ResourceQueue
 from dkist_processing_core.task import TaskBase
 
-
 task_type_hint = Type[TaskBase]
 upstreams_type_hint = list[task_type_hint] | task_type_hint | None
 
@@ -50,9 +50,10 @@ class Node:
     def operator(self) -> BashOperator:
         """Native engine node."""
         from datetime import timedelta
-        from dkist_processing_core.failure_callback import chat_ops_notification
         from functools import partial
 
+        from dkist_processing_core.failure_callback import chat_ops_notification
+
         return eval(self.operator_definition)
 
     @property
@@ -93,6 +94,8 @@ class Node:
     def bash_script(self) -> str:
         """Format bash script for the BashOperator."""
         command = f"""{self.install_command}
+echo Virtual Python Environment Packages
+pip list
 {self.run_command}"""
         return self.bash_template(command)
 
@@ -115,7 +118,7 @@ echo Host Python Environment i.e. system-site-packages
 python3 -m pip install --upgrade --user pip
 pip list
 echo Creating Virtual Environment
-python3 -m venv --system-site-packages .task_venv
+python3 -m venv .task_venv
 echo Activate Environment
 . .task_venv/bin/activate
 echo Python Interpreter Location

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/resource_queue.py

@@ -1,4 +1,5 @@
 """Resource queue names for Workflow Node specification which specify different resource needs."""
+
 from enum import StrEnum
 
 

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/tests/conftest.py

@@ -1,4 +1,5 @@
 """Global test fixtures."""
+
 from contextlib import contextmanager
 from pathlib import Path
 from shutil import rmtree

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/tests/invalid_workflow_for_docker_multi_category/workflow.py

@@ -1,8 +1,8 @@
-"""Example invalid workflow for """
+"""Example invalid workflow for"""
+
 from dkist_processing_core import Workflow
 from dkist_processing_core.tests.task_example import Task
 
-
 category_a = Workflow(
     input_data="test-data",
     output_data="invalid",

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/tests/task_example.py

@@ -1,4 +1,5 @@
 """Example task subclass used in the tests."""
+
 from dkist_processing_core import TaskBase
 
 

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/tests/test_build_utils.py

@@ -1,4 +1,5 @@
 """Tests for the build utils."""
+
 import os
 import subprocess
 from pathlib import Path

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/tests/test_export.py

@@ -1,4 +1,5 @@
 """Test minimal dependency dag export (used for scheduler deployments)."""
+
 import nbformat
 import pytest
 from nbconvert import PythonExporter

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/tests/test_failure_callback.py

@@ -1,12 +1,13 @@
 """Test for the failure callback function."""
+
 import pytest
 from talus import DurableProducer
 
+from dkist_processing_core.failure_callback import RecipeRunFailureMessage
 from dkist_processing_core.failure_callback import chat_ops_notification
 from dkist_processing_core.failure_callback import parse_dag_run_id_from_context
 from dkist_processing_core.failure_callback import parse_log_url_from_context
 from dkist_processing_core.failure_callback import recipe_run_failure_message_producer_factory
-from dkist_processing_core.failure_callback import RecipeRunFailureMessage
 
 
 @pytest.fixture()

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/tests/test_node.py

@@ -1,4 +1,5 @@
 """Tests of the node.py module."""
+
 import subprocess
 from subprocess import CalledProcessError
 from typing import Callable

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/tests/test_task.py

@@ -1,4 +1,5 @@
 """Tests for the TaskBase functionality."""
+
 import pytest
 
 from dkist_processing_core.config import core_configurations

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/tests/test_workflow.py

@@ -1,4 +1,5 @@
 """Tests for the Workflow abstraction."""
+
 import json
 from typing import Callable
 

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/tests/valid_workflow_package/workflow.py

@@ -1,4 +1,5 @@
 """Example valid workflow."""
+
 from dkist_processing_core import Workflow
 from dkist_processing_core.tests.task_example import Task
 from dkist_processing_core.tests.task_example import Task2

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/tests/zero_node_workflow_package/workflow.py

@@ -1,4 +1,5 @@
 """Workflow containing zero nodes."""
+
 from dkist_processing_core import Workflow
 
 zero_node_workflow = Workflow(

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/workflow.py

@@ -1,4 +1,5 @@
 """Abstraction layer to construct a workflow as an airflow DAG."""
+
 import json
 import string
 from graphlib import TopologicalSorter

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: dkist-processing-core
-Version: 5.1.0
+Version: 5.2.0
 Summary: Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -10,10 +10,10 @@ Project-URL: Documentation, https://docs.dkist.nso.edu/projects/core
 Project-URL: Help, https://nso.atlassian.net/servicedesk/customer/portal/5
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.11
-Requires-Python: >=3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.12
 Description-Content-Type: text/x-rst
-Requires-Dist: apache-airflow[celery,postgres]==2.10.5
+Requires-Dist: apache-airflow[celery,postgres]==2.11.0
 Requires-Dist: elastic-apm<7.0.0
 Requires-Dist: requests>=2.23
 Requires-Dist: talus<2.0,>=1.1.0
@@ -43,6 +43,8 @@ Requires-Dist: sunpy; extra == "docs"
 dkist-processing-core
 =====================
 
+|codecov|
+
 Overview
 --------
 The dkist-processing-core package provides an abstraction layer between the dkist data processing code, the workflow
@@ -247,3 +249,6 @@ then the Bitbucket pipeline will fail. To be able to use the same tag you must d
     # Re-tag with the same version
     git tag vWHATEVER.THE.VERSION
     git push --tags origin main
+
+.. |codecov| image:: https://codecov.io/bb/dkistdc/dkist-processing-core/graph/badge.svg?token=SB18SCBJ8Q
+ :target: https://codecov.io/bb/dkistdc/dkist-processing-core

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core.egg-info/requires.txt

@@ -1,4 +1,4 @@
-apache-airflow[celery,postgres]==2.10.5
+apache-airflow[celery,postgres]==2.11.0
 elastic-apm<7.0.0
 requests>=2.23
 talus<2.0,>=1.1.0

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/docs/conf.py

@@ -1,4 +1,5 @@
 """Configuration file for the Sphinx documentation builder."""
+
 # -- stdlib imports ------------------------------------------------------------
 import importlib
 import sys

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=64", "setuptools_scm>=8"]
+requires = ["setuptools>=64", "setuptools_scm>=8", "wheel", "build"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools.packages]
@@ -10,7 +10,7 @@ find = {}
 [project]
 dynamic = ["version"]
 name = "dkist-processing-core"
-requires-python = ">=3.11"
+requires-python = ">=3.12"
 description = "Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow"
 readme = "README.rst"
 authors = [ {name = "NSO / AURA", email = "dkistdc@nso.edu"} ]
@@ -18,10 +18,10 @@ license = {text = "BSD-3-Clause"}
 classifiers = [
   "Programming Language :: Python",
   "Programming Language :: Python :: 3",
-  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
-  "apache-airflow[postgres, celery] == 2.10.5",
+  "apache-airflow[postgres, celery] == 2.11.0",
   "elastic-apm < 7.0.0",
   "requests >= 2.23",
   "talus >= 1.1.0, <2.0",
@@ -107,3 +107,12 @@ show_missing = true
       directory = "doc"
       name = "Documentation"
       showcontent = true
+
+[tool.black]
+line-length = 100
+
+[tool.isort]
+profile = "black"
+force_single_line = true
+combine_as_imports = false
+line_length = 100

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/task.py

@@ -4,6 +4,7 @@ Base class that is used to wrap the various DAG task methods.
 It provides support for user-defined setup and cleanup, task monitoring using Elastic APM,
 standardized logging and exception handling.
 """
+
 import logging
 from abc import ABC
 from abc import abstractmethod
@@ -13,7 +14,6 @@ import elasticapm
 
 from dkist_processing_core.config import core_configurations
 
-
 __all__ = ["TaskBase"]
 
 logger = logging.getLogger(__name__)

{dkist_processing_core-5.1.0 → dkist_processing_core-5.2.0}/dkist_processing_core/tests/invalid_workflow_cyclic/workflow.py

@@ -1,10 +1,10 @@
 """Example invalid workflow."""
+
 from dkist_processing_core import Workflow
 from dkist_processing_core.tests.task_example import Task
 from dkist_processing_core.tests.task_example import Task2
 from dkist_processing_core.tests.task_example import Task3
 
-
 # |<--------------------|
 # |-->Task -> Task2 --> |