dkist-processing-core 7.2.2rc1__tar.gz → 7.3.0__tar.gz

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.3.0}/CHANGELOG.rst

@@ -1,3 +1,28 @@
+v7.3.0 (2026-05-06)
+===================
+
+Features
+--------
+
+- Add utility for working with the notebook build target which supports waiting for a notebook to autosave. (`#76 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/76>`__)
+- Add utility for working with the notebook build target which supports exporting a notebook as markdown. (`#76 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/76>`__)
+
+
+Bugfixes
+--------
+
+- Prevent DAG version churn caused by dynamic `start_date` values by ensuring DAG metadata is deterministic. (`#78 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/78>`__)
+
+
+Misc
+----
+
+- Add 'IS_NOTEBOOK_EXECUTION_ENVIRONMENT' as an environment variable to indicate when the execution environment is a Jupyter notebook. (`#72 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/72>`__)
+- Refactor the dag naming function to be accessible outside of the context of a Workflow instance. (`#77 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/77>`__)
+- Address deprecated import of airflow.exceptions.DuplicateTaskIdFound using airflow.sdk.exceptions.DuplicateTaskIdFound instead. (`#79 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/79>`__)
+- Upgrade `apache-airflow` to version 3.2.1. (`#79 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/79>`__)
+
+
 v7.2.1 (2026-04-16)
 ===================
 

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-core
-Version: 7.2.2rc1
+Version: 7.3.0
 Summary: Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -13,10 +13,9 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.13
 Description-Content-Type: text/x-rst
-Requires-Dist: apache-airflow[celery,postgres]==3.1.8
+Requires-Dist: apache-airflow[celery,postgres]==3.2.1
 Requires-Dist: requests>=2.23
 Requires-Dist: talus<2.0,>=1.3.4
-Requires-Dist: pendulum
 Requires-Dist: nbformat>=5.9.2
 Requires-Dist: notebook<8.0,>=7.5.5
 Requires-Dist: nbconvert<8.0,>=7.16.6
@@ -28,8 +27,6 @@ Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: pytest-mock; extra == "test"
 Requires-Dist: jinja2; extra == "test"
 Requires-Dist: towncrier; extra == "test"
-Requires-Dist: nbconvert; extra == "test"
-Requires-Dist: ipython; extra == "test"
 Provides-Extra: docs
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: sphinx-astropy; extra == "docs"
@@ -178,6 +175,8 @@ Environment Variables
      - annotation=int required=False default=15 description='Timeout for pip installs in seconds.'
    * - PIP_RETRIES
      - annotation=int required=False default=5 description='Number of retries for pip installs.'
+   * - IS_NOTEBOOK_EXECUTION_ENVIRONMENT
+     - annotation=bool required=False default=False description='Indication of whether the code is running in a notebook execution environment.' examples=[True, False]
 
 Development
 -----------

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.3.0}/README.rst

@@ -135,6 +135,8 @@ Environment Variables
      - annotation=int required=False default=15 description='Timeout for pip installs in seconds.'
    * - PIP_RETRIES
      - annotation=int required=False default=5 description='Number of retries for pip installs.'
+   * - IS_NOTEBOOK_EXECUTION_ENVIRONMENT
+     - annotation=bool required=False default=False description='Indication of whether the code is running in a notebook execution environment.' examples=[True, False]
 
 Development
 -----------

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.3.0}/dkist_processing_core/build_utils.py

@@ -119,10 +119,7 @@ class NotebookDockerfile:
             "COPY . /app",
             "WORKDIR /app",
             "RUN python -m pip install -U pip",
-            "RUN pip install notebook",
-            "RUN pip freeze | grep notebook= > constraints.txt",
-            "RUN cat constraints.txt",
-            "RUN python -m pip install -c constraints.txt .",
+            "RUN python -m pip install .",
         ]
 
     @property

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.3.0}/dkist_processing_core/notebook_utils.py

@@ -2,19 +2,18 @@
 
 import time
 from pathlib import Path
+from uuid import uuid4
 
 import nbformat
 from nbconvert import MarkdownExporter
 
-DEFAULT_EXPORT_SENTINEL = "MPW_EXPORT_READY"
-
 
 def wait_for_notebook_save(
-    notebook_path: Path | str,
-    sentinel: str = DEFAULT_EXPORT_SENTINEL,
-    attempts=120,
-    delay_s=1,
-):
+    notebook_path: Path,
+    sentinel: str | None = None,
+    attempts: int = 120,
+    delay_s: int = 1,
+) -> None:
     """
     Wait until the notebook on disk contains the sentinel output.
 
@@ -37,28 +36,20 @@ def wait_for_notebook_save(
     -------
     None if the sentinel is found within the notebook outputs, otherwise raises a TimeoutError after the specified number of attempts.
     """
+    sentinel = sentinel or uuid4().hex
+
     print(f"Waiting for notebook to save with sentinel '{sentinel}'...", flush=True)
-    for attempt in range(attempts):
-        with open(notebook_path, "r", encoding="utf-8") as f:
-            nb = nbformat.read(f, as_version=nbformat.NO_CONVERT)
-
-        # Search outputs
-        for cell in nb.cells:
-            if cell.cell_type == "code":
-                for output in cell.get("outputs", []):
-                    text = ""
-
-                    if output.output_type == "stream":
-                        text = output.get("text", "")
-                    elif output.output_type in ("execute_result", "display_data"):
-                        text = str(output.get("data", ""))
-                    if sentinel in text:
-                        return
+
+    for _ in range(attempts):
+        text = notebook_path.read_text(encoding="utf-8")
+        if sentinel in text:
+            return
         time.sleep(delay_s)
+
     raise TimeoutError("Notebook did not save updated outputs in time")
 
 
-def export_notebook_by_path(notebook_path) -> bytes:
+def export_notebook_by_path(notebook_path: Path) -> bytes:
     """
     Export the notebook at the given path to markdown format and return the resulting bytes.
 
@@ -66,10 +57,13 @@ def export_notebook_by_path(notebook_path) -> bytes:
     ----------
     notebook_path
         The path to the notebook file to export.
+
+    Returns
+    -------
+    The exported notebook in markdown format as bytes.
     """
-    with open(notebook_path, "r", encoding="utf-8") as f:
-        nb = nbformat.read(f, as_version=4)
+    notebook = nbformat.read(notebook_path, as_version=nbformat.NO_CONVERT)
 
     exporter = MarkdownExporter()
-    body, _ = exporter.from_notebook_node(nb)
+    body, _ = exporter.from_notebook_node(notebook)
     return body.encode("utf-8")

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.3.0}/dkist_processing_core/tests/test_build_utils.py

@@ -7,7 +7,7 @@ from shutil import rmtree
 
 import pytest
 from airflow.exceptions import AirflowException
-from airflow.exceptions import DuplicateTaskIdFound
+from airflow.sdk.exceptions import DuplicateTaskIdFound
 
 from dkist_processing_core.build_utils import export_dags
 from dkist_processing_core.build_utils import export_notebook_dockerfile

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.3.0}/dkist_processing_core/tests/test_notebook_utils.py

@@ -1,6 +1,7 @@
 """Tests for the notebook_utils module"""
 
 from pathlib import Path
+from uuid import uuid4
 
 import nbformat
 import pytest
@@ -9,7 +10,6 @@ from nbformat.v4 import new_markdown_cell
 from nbformat.v4 import new_notebook
 from nbformat.v4 import new_output
 
-from dkist_processing_core.notebook_utils import DEFAULT_EXPORT_SENTINEL
 from dkist_processing_core.notebook_utils import export_notebook_by_path
 from dkist_processing_core.notebook_utils import wait_for_notebook_save
 
@@ -28,7 +28,12 @@ def notebook_path_factory(tmp_path: Path):
 
 
 @pytest.fixture
-def notebook_with_sentinel_output(notebook_path_factory) -> Path:
+def notebook_sentinel() -> str:
+    return uuid4().hex
+
+
+@pytest.fixture
+def notebook_with_sentinel_output(notebook_path_factory, notebook_sentinel) -> Path:
     return notebook_path_factory(
         cells=[
             new_markdown_cell("# Title"),
@@ -38,7 +43,7 @@ def notebook_with_sentinel_output(notebook_path_factory) -> Path:
                     new_output(
                         output_type="stream",
                         name="stdout",
-                        text=f"before {DEFAULT_EXPORT_SENTINEL} after",
+                        text=f"before {notebook_sentinel} after",
                     )
                 ],
             ),
@@ -65,10 +70,12 @@ def notebook_without_sentinel_output(notebook_path_factory) -> Path:
 
 
 def test_wait_for_notebook_save_succeeds_when_sentinel_in_code_cell_output(
-    notebook_with_sentinel_output,
+    notebook_with_sentinel_output, notebook_sentinel
 ):
     # no error raised
-    wait_for_notebook_save(notebook_with_sentinel_output, attempts=1, delay_s=0)
+    wait_for_notebook_save(
+        notebook_with_sentinel_output, sentinel=notebook_sentinel, attempts=1, delay_s=0
+    )
 
 
 def test_wait_for_notebook_save_times_out_when_sentinel_missing(
@@ -78,7 +85,9 @@ def test_wait_for_notebook_save_times_out_when_sentinel_missing(
         wait_for_notebook_save(notebook_without_sentinel_output, attempts=1, delay_s=0)
 
 
-def test_export_notebook_by_path_returns_markdown_bytes(notebook_with_sentinel_output) -> None:
+def test_export_notebook_by_path_returns_markdown_bytes(
+    notebook_with_sentinel_output, notebook_sentinel
+) -> None:
     exported = export_notebook_by_path(notebook_with_sentinel_output)
 
     assert isinstance(exported, bytes)
@@ -87,4 +96,4 @@ def test_export_notebook_by_path_returns_markdown_bytes(notebook_with_sentinel_o
 
     assert "# Title" in output  # markdown
     assert "print('ready')" in output  # code
-    assert DEFAULT_EXPORT_SENTINEL in output  # output
+    assert notebook_sentinel in output  # output

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.3.0}/dkist_processing_core/workflow.py

@@ -13,7 +13,7 @@ from dkist_processing_core.node import task_type_hint
 from dkist_processing_core.node import upstreams_type_hint
 from dkist_processing_core.resource_queue import ResourceQueue
 
-__all__ = ["Workflow", "workflow_name_from_details"]
+__all__ = ["Workflow", "workflow_name_from_details", "dag_name"]
 
 MAXIMUM_ALLOWED_WORKFLOW_NAME_LENGTH = 100
 
@@ -47,12 +47,11 @@ def _check_dag_name_characters(dag_name: str):
 
     Raise error if non-allowed characters are found.
     """
-    allowed_chars = [c for c in string.ascii_letters] + ["-", ".", "_"] + [n for n in string.digits]
+    allowed_special_characters = "-._"
+    allowed_chars = string.ascii_letters + string.digits + allowed_special_characters
     if not all([char in allowed_chars for char in dag_name]):
         raise ValueError(
-            f"Dag name {dag_name} contains invalid characters. "
-            f"Only ascii letters and the dash, dot, and "
-            f"underscore symbols are permitted."
+            f"Dag name {dag_name} contains invalid characters. Only {allowed_chars} are permitted."
         )
 
 
@@ -163,11 +162,10 @@ class Workflow:
     @property
     def dag_definition(self) -> str:
         """Return the string representation of the DAG object instantiation."""
-        return f"DAG(dag_id='{self.dag_name}', start_date=pendulum.today('UTC').add(days=-2), schedule=None, catchup=False, tags={self.dag_tags})"
+        return f"DAG(dag_id='{self.dag_name}', schedule=None, catchup=False, tags={self.dag_tags})"
 
     def initialize_local_dag(self):
         """Create a local instance of the DAG object."""
-        import pendulum
         from airflow import DAG
 
         return eval(self.dag_definition)
@@ -226,7 +224,7 @@ class Workflow:
             f.write("\n")
         return workflow_py
 
-    def export_notebook(self, path: str | Path | None = None):
+    def export_notebook(self, path: str | Path | None = None) -> Path:
         """Render the workflow as a jupyter notebook."""
         path = path or "notebooks/"
         path = Path(path)
@@ -241,7 +239,7 @@ class Workflow:
             nbf.write(nb, f)
         return notebook_ipynb
 
-    def topological_sort(self) -> [Node]:
+    def topological_sort(self) -> list[Node]:
         """Use a topological sort to find a valid linear order for task execution."""
         node_task_names = {node.task.__name__: node for node in self.nodes}
         node_upstream_tasks = {node.task: node.upstreams for node in self.nodes}
@@ -258,7 +256,6 @@ class Workflow:
             "",
             "from airflow import DAG",
             "from airflow.providers.standard.operators.bash import BashOperator",
-            "import pendulum",
             "",
             "from dkist_processing_core.failure_callback import chat_ops_notification",
             "",

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.3.0}/dkist_processing_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-core
-Version: 7.2.2rc1
+Version: 7.3.0
 Summary: Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -13,10 +13,9 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.13
 Description-Content-Type: text/x-rst
-Requires-Dist: apache-airflow[celery,postgres]==3.1.8
+Requires-Dist: apache-airflow[celery,postgres]==3.2.1
 Requires-Dist: requests>=2.23
 Requires-Dist: talus<2.0,>=1.3.4
-Requires-Dist: pendulum
 Requires-Dist: nbformat>=5.9.2
 Requires-Dist: notebook<8.0,>=7.5.5
 Requires-Dist: nbconvert<8.0,>=7.16.6
@@ -28,8 +27,6 @@ Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: pytest-mock; extra == "test"
 Requires-Dist: jinja2; extra == "test"
 Requires-Dist: towncrier; extra == "test"
-Requires-Dist: nbconvert; extra == "test"
-Requires-Dist: ipython; extra == "test"
 Provides-Extra: docs
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: sphinx-astropy; extra == "docs"
@@ -178,6 +175,8 @@ Environment Variables
      - annotation=int required=False default=15 description='Timeout for pip installs in seconds.'
    * - PIP_RETRIES
      - annotation=int required=False default=5 description='Number of retries for pip installs.'
+   * - IS_NOTEBOOK_EXECUTION_ENVIRONMENT
+     - annotation=bool required=False default=False description='Indication of whether the code is running in a notebook execution environment.' examples=[True, False]
 
 Development
 -----------

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.3.0}/dkist_processing_core.egg-info/SOURCES.txt

@@ -7,10 +7,6 @@ README.rst
 bitbucket-pipelines.yml
 pyproject.toml
 changelog/.gitempty
-changelog/72.misc.rst
-changelog/76.feature.1.rst
-changelog/76.feature.2.rst
-changelog/77.misc.rst
 dkist_processing_core/__init__.py
 dkist_processing_core/build_utils.py
 dkist_processing_core/config.py

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.3.0}/dkist_processing_core.egg-info/requires.txt

@@ -1,7 +1,6 @@
-apache-airflow[celery,postgres]==3.1.8
+apache-airflow[celery,postgres]==3.2.1
 requests>=2.23
 talus<2.0,>=1.3.4
-pendulum
 nbformat>=5.9.2
 notebook<8.0,>=7.5.5
 nbconvert<8.0,>=7.16.6
@@ -25,5 +24,3 @@ pytest-cov
 pytest-mock
 jinja2
 towncrier
-nbconvert
-ipython

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.3.0}/pyproject.toml

@@ -21,10 +21,9 @@ classifiers = [
   "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
-  "apache-airflow[postgres, celery] == 3.1.8",
+  "apache-airflow[postgres, celery] == 3.2.1",
   "requests >= 2.23",
   "talus >= 1.3.4, <2.0",
-  "pendulum",
   "nbformat >= 5.9.2",
   "notebook >= 7.5.5, < 8.0",
   "nbconvert >= 7.16.6, < 8.0",
@@ -45,8 +44,6 @@ test = [
   "pytest-mock",
   "jinja2",
   "towncrier",
-  "nbconvert",
-  "ipython",
 ]
 docs = [
   "sphinx",

dkist_processing_core-7.2.2rc1/changelog/72.misc.rst

@@ -1 +0,0 @@
-Add an environment variable to indicate when the execution environment is a Jupyter notebook.

dkist_processing_core-7.2.2rc1/changelog/76.feature.1.rst

@@ -1 +0,0 @@
-Add utility for working with the notebook build target which supports waiting for a notebook to autosave.

dkist_processing_core-7.2.2rc1/changelog/76.feature.2.rst

@@ -1 +0,0 @@
-Add utility for working with the notebook build target which supports exporting a notebook as markdown.

dkist_processing_core-7.2.2rc1/changelog/77.misc.rst

@@ -1 +0,0 @@
-Refactor the dag naming function to be accessible outside of the context of a Workflow instance.