dkist-processing-core 7.2.1rc1__tar.gz → 7.2.2rc2__tar.gz

{dkist_processing_core-7.2.1rc1 → dkist_processing_core-7.2.2rc2}/CHANGELOG.rst

@@ -1,3 +1,13 @@
+v7.2.1 (2026-04-16)
+===================
+
+Misc
+----
+
+- Configure pyproject.toml to turn warnings raised during testing into errors. (`#74 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/74>`__)
+- Upgrade Read the Docs LTS build image to Ubuntu 24.04. (`#75 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/75>`__)
+
+
 v7.1.0 (2026-04-01)
 ===================
 

{dkist_processing_core-7.2.1rc1 → dkist_processing_core-7.2.2rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-core
-Version: 7.2.1rc1
+Version: 7.2.2rc2
 Summary: Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -18,6 +18,8 @@ Requires-Dist: requests>=2.23
 Requires-Dist: talus<2.0,>=1.3.4
 Requires-Dist: pendulum
 Requires-Dist: nbformat>=5.9.2
+Requires-Dist: notebook<8.0,>=7.5.5
+Requires-Dist: nbconvert<8.0,>=7.16.6
 Requires-Dist: dkist-service-configuration<5.0,>=4.3.0
 Requires-Dist: pydantic>2.0
 Provides-Extra: test

dkist_processing_core-7.2.2rc2/changelog/72.misc.rst

@@ -0,0 +1 @@
+Add an environment variable to indicate when the execution environment is a Jupyter notebook.

dkist_processing_core-7.2.2rc2/changelog/76.feature.1.rst

@@ -0,0 +1 @@
+Add utility for working with the notebook build target which supports waiting for a notebook to autosave.

dkist_processing_core-7.2.2rc2/changelog/76.feature.2.rst

@@ -0,0 +1 @@
+Add utility for working with the notebook build target which supports exporting a notebook as markdown.

dkist_processing_core-7.2.2rc2/changelog/77.misc.rst

@@ -0,0 +1 @@
+Refactor the dag naming function to be accessible outside of the context of a Workflow instance.

{dkist_processing_core-7.2.1rc1 → dkist_processing_core-7.2.2rc2}/dkist_processing_core/build_utils.py

@@ -115,13 +115,11 @@ class NotebookDockerfile:
     def setup(self) -> list[str]:
         """Environment setup lines."""
         return [
+            "ENV IS_NOTEBOOK_EXECUTION_ENVIRONMENT=true",
             "COPY . /app",
             "WORKDIR /app",
             "RUN python -m pip install -U pip",
-            "RUN pip install notebook",
-            "RUN pip freeze | grep notebook= > constraints.txt",
-            "RUN cat constraints.txt",
-            "RUN python -m pip install -c constraints.txt .",
+            "RUN python -m pip install .",
         ]
 
     @property

{dkist_processing_core-7.2.1rc1 → dkist_processing_core-7.2.2rc2}/dkist_processing_core/config.py

@@ -27,6 +27,11 @@ class DKISTProcessingCoreConfiguration(InstrumentedMeshServiceConfigurationBase)
     )
     pip_timeout: int = Field(default=15, description="Timeout for pip installs in seconds.")
     pip_retries: int = Field(default=5, description="Number of retries for pip installs.")
+    is_notebook_execution_environment: bool = Field(
+        default=False,
+        description="Indication of whether the code is running in a notebook execution environment.",
+        examples=[True, False],
+    )
 
     @property
     def isb_mesh_service(self) -> MeshService:

dkist_processing_core-7.2.2rc2/dkist_processing_core/notebook_utils.py

@@ -0,0 +1,70 @@
+"""Utilities for working with the notebook execution environment."""
+
+import time
+from pathlib import Path
+from uuid import uuid4
+
+import nbformat
+from nbconvert import MarkdownExporter
+
+
+def wait_for_notebook_save(
+    notebook_path: Path,
+    sentinel: str | None = None,
+    attempts: int = 120,
+    delay_s: int = 1,
+) -> None:
+    """
+    Wait until the notebook on disk contains the sentinel output.
+
+    Parameters
+    ----------
+    notebook_path
+        The path to the notebook file to check for the sentinel output.
+
+    sentinel
+        The string to search for in the notebook outputs to confirm that the notebook has been saved with the expected output.
+
+    attempts
+        The number of times to check the notebook for the sentinel output before giving up and raising a
+        TimeoutError.
+
+    delay_s
+        The number of seconds to wait between attempts to check the notebook for the sentinel output.
+
+    Returns
+    -------
+    None if the sentinel is found within the notebook outputs, otherwise raises a TimeoutError after the specified number of attempts.
+    """
+    sentinel = sentinel or uuid4().hex
+
+    print(f"Waiting for notebook to save with sentinel '{sentinel}'...", flush=True)
+
+    for _ in range(attempts):
+        text = notebook_path.read_text(encoding="utf-8")
+        if sentinel in text:
+            return
+        time.sleep(delay_s)
+
+    raise TimeoutError("Notebook did not save updated outputs in time")
+
+
+def export_notebook_by_path(notebook_path: Path) -> bytes:
+    """
+    Export the notebook at the given path to markdown format and return the resulting bytes.
+
+    Parameters
+    ----------
+    notebook_path
+        The path to the notebook file to export.
+
+    Returns
+    -------
+    The exported notebook in markdown format as bytes.
+    """
+    notebook_contents = notebook_path.read_text(encoding="utf-8")
+    notebook = nbformat.reads(notebook_contents, as_version=nbformat.NO_CONVERT)
+
+    exporter = MarkdownExporter()
+    body, _ = exporter.from_notebook_node(notebook)
+    return body.encode("utf-8")

dkist_processing_core-7.2.2rc2/dkist_processing_core/tests/test_notebook_utils.py

@@ -0,0 +1,99 @@
+"""Tests for the notebook_utils module"""
+
+from pathlib import Path
+from uuid import uuid4
+
+import nbformat
+import pytest
+from nbformat.v4 import new_code_cell
+from nbformat.v4 import new_markdown_cell
+from nbformat.v4 import new_notebook
+from nbformat.v4 import new_output
+
+from dkist_processing_core.notebook_utils import export_notebook_by_path
+from dkist_processing_core.notebook_utils import wait_for_notebook_save
+
+
+@pytest.fixture
+def notebook_path_factory(tmp_path: Path):
+    def _make_notebook(*, cells=None) -> Path:
+        notebook_path = tmp_path / "test.ipynb"
+        cells = cells or []
+        nb = new_notebook(cells=cells)
+        with notebook_path.open("w", encoding="utf-8") as f:
+            nbformat.write(nb, f)
+        return notebook_path
+
+    return _make_notebook
+
+
+@pytest.fixture
+def notebook_sentinel() -> str:
+    return uuid4().hex
+
+
+@pytest.fixture
+def notebook_with_sentinel_output(notebook_path_factory, notebook_sentinel) -> Path:
+    return notebook_path_factory(
+        cells=[
+            new_markdown_cell("# Title"),
+            new_code_cell(
+                source="print('ready')",
+                outputs=[
+                    new_output(
+                        output_type="stream",
+                        name="stdout",
+                        text=f"before {notebook_sentinel} after",
+                    )
+                ],
+            ),
+        ]
+    )
+
+
+@pytest.fixture
+def notebook_without_sentinel_output(notebook_path_factory) -> Path:
+    return notebook_path_factory(
+        cells=[
+            new_code_cell(
+                source="print('not ready')",
+                outputs=[
+                    new_output(
+                        output_type="stream",
+                        name="stdout",
+                        text="some other output",
+                    )
+                ],
+            )
+        ]
+    )
+
+
+def test_wait_for_notebook_save_succeeds_when_sentinel_in_code_cell_output(
+    notebook_with_sentinel_output, notebook_sentinel
+):
+    # no error raised
+    wait_for_notebook_save(
+        notebook_with_sentinel_output, sentinel=notebook_sentinel, attempts=1, delay_s=0
+    )
+
+
+def test_wait_for_notebook_save_times_out_when_sentinel_missing(
+    notebook_without_sentinel_output,
+):
+    with pytest.raises(TimeoutError, match="Notebook did not save updated outputs in time"):
+        wait_for_notebook_save(notebook_without_sentinel_output, attempts=1, delay_s=0)
+
+
+def test_export_notebook_by_path_returns_markdown_bytes(
+    notebook_with_sentinel_output, notebook_sentinel
+) -> None:
+    exported = export_notebook_by_path(notebook_with_sentinel_output)
+
+    assert isinstance(exported, bytes)
+
+    output = exported.decode("utf-8")
+
+    assert "# Title" in output  # markdown
+    assert "print('ready')" in output  # code
+    assert notebook_sentinel in output  # output

{dkist_processing_core-7.2.1rc1 → dkist_processing_core-7.2.2rc2}/dkist_processing_core/tests/test_workflow.py

@@ -9,6 +9,7 @@ from airflow import DAG
 from dkist_processing_core import ResourceQueue
 from dkist_processing_core import Workflow
 from dkist_processing_core.workflow import MAXIMUM_ALLOWED_WORKFLOW_NAME_LENGTH
+from dkist_processing_core.workflow import _check_dag_name_characters
 from dkist_processing_core.workflow import workflow_name_from_details
 
 
@@ -165,9 +166,9 @@ def test_check_dag_name_characters():
     When: checking if it is a valid airflow name or not
     Then: correctly identify valid and invalid names
     """
-    Workflow.check_dag_name_characters(dag_name="This_dag_name_is_valid")
+    _check_dag_name_characters(dag_name="This_dag_name_is_valid")
     with pytest.raises(ValueError):
-        Workflow.check_dag_name_characters(dag_name="Invalid*dag*name")
+        _check_dag_name_characters(dag_name="Invalid*dag*name")
 
 
 @pytest.mark.parametrize(

{dkist_processing_core-7.2.1rc1 → dkist_processing_core-7.2.2rc2}/dkist_processing_core/workflow.py

@@ -13,7 +13,7 @@ from dkist_processing_core.node import task_type_hint
 from dkist_processing_core.node import upstreams_type_hint
 from dkist_processing_core.resource_queue import ResourceQueue
 
-__all__ = ["Workflow", "workflow_name_from_details"]
+__all__ = ["Workflow", "workflow_name_from_details", "dag_name"]
 
 MAXIMUM_ALLOWED_WORKFLOW_NAME_LENGTH = 100
 
@@ -34,6 +34,35 @@ def workflow_name_from_details(
     return workflow_name
 
 
+def _check_dag_name_characters(dag_name: str):
+    """
+    Figure out if the dag name is an Airflow-allowed name.
+
+    Can only contain
+    * ascii letters
+    * numbers
+    * dash (-)
+    * dot (.)
+    * underscore (_)
+
+    Raise error if non-allowed characters are found.
+    """
+    allowed_chars = [c for c in string.ascii_letters] + ["-", ".", "_"] + [n for n in string.digits]
+    if not all([char in allowed_chars for char in dag_name]):
+        raise ValueError(
+            f"Dag name {dag_name} contains invalid characters. "
+            f"Only ascii letters and the dash, dot, and "
+            f"underscore symbols are permitted."
+        )
+
+
+def dag_name(workflow_name: str, workflow_version: str) -> str:
+    """Return the dag name created from its constituent parts."""
+    result = f"{workflow_name}_{workflow_version}"
+    _check_dag_name_characters(result)  # raise an error if in valid
+    return result
+
+
 class Workflow:
     """
     Abstraction to create a workflow in 1 or more target execution environment.
@@ -116,33 +145,7 @@ class Workflow:
     @property
     def dag_name(self) -> str:
         """Return the dag name created from its constituent parts."""
-        result = f"{self.workflow_name}_{self.workflow_version}"
-        self.check_dag_name_characters(result)  # raise an error if in valid
-        return result
-
-    @staticmethod
-    def check_dag_name_characters(dag_name: str):
-        """
-        Figure out if the dag name is an Airflow-allowed name.
-
-        Can only contain
-        * ascii letters
-        * numbers
-        * dash (-)
-        * dot (.)
-        * underscore (_)
-
-        Raise error if non-allowed characters are found.
-        """
-        allowed_chars = (
-            [c for c in string.ascii_letters] + ["-", ".", "_"] + [n for n in string.digits]
-        )
-        if not all([char in allowed_chars for char in dag_name]):
-            raise ValueError(
-                f"Dag name {dag_name} contains invalid characters. "
-                f"Only ascii letters and the dash, dot, and "
-                f"underscore symbols are permitted."
-            )
+        return dag_name(workflow_name=self.workflow_name, workflow_version=self.workflow_version)
 
     @property
     def dag_tags(self) -> str:
@@ -223,7 +226,7 @@ class Workflow:
             f.write("\n")
         return workflow_py
 
-    def export_notebook(self, path: str | Path | None = None):
+    def export_notebook(self, path: str | Path | None = None) -> Path:
         """Render the workflow as a jupyter notebook."""
         path = path or "notebooks/"
         path = Path(path)
@@ -238,7 +241,7 @@ class Workflow:
             nbf.write(nb, f)
         return notebook_ipynb
 
-    def topological_sort(self) -> [Node]:
+    def topological_sort(self) -> list[Node]:
         """Use a topological sort to find a valid linear order for task execution."""
         node_task_names = {node.task.__name__: node for node in self.nodes}
         node_upstream_tasks = {node.task: node.upstreams for node in self.nodes}

{dkist_processing_core-7.2.1rc1 → dkist_processing_core-7.2.2rc2}/dkist_processing_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-core
-Version: 7.2.1rc1
+Version: 7.2.2rc2
 Summary: Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -18,6 +18,8 @@ Requires-Dist: requests>=2.23
 Requires-Dist: talus<2.0,>=1.3.4
 Requires-Dist: pendulum
 Requires-Dist: nbformat>=5.9.2
+Requires-Dist: notebook<8.0,>=7.5.5
+Requires-Dist: nbconvert<8.0,>=7.16.6
 Requires-Dist: dkist-service-configuration<5.0,>=4.3.0
 Requires-Dist: pydantic>2.0
 Provides-Extra: test

{dkist_processing_core-7.2.1rc1 → dkist_processing_core-7.2.2rc2}/dkist_processing_core.egg-info/SOURCES.txt

@@ -7,12 +7,16 @@ README.rst
 bitbucket-pipelines.yml
 pyproject.toml
 changelog/.gitempty
-changelog/75.misc.rst
+changelog/72.misc.rst
+changelog/76.feature.1.rst
+changelog/76.feature.2.rst
+changelog/77.misc.rst
 dkist_processing_core/__init__.py
 dkist_processing_core/build_utils.py
 dkist_processing_core/config.py
 dkist_processing_core/failure_callback.py
 dkist_processing_core/node.py
+dkist_processing_core/notebook_utils.py
 dkist_processing_core/resource_queue.py
 dkist_processing_core/task.py
 dkist_processing_core/workflow.py
@@ -28,6 +32,7 @@ dkist_processing_core/tests/test_build_utils.py
 dkist_processing_core/tests/test_export.py
 dkist_processing_core/tests/test_failure_callback.py
 dkist_processing_core/tests/test_node.py
+dkist_processing_core/tests/test_notebook_utils.py
 dkist_processing_core/tests/test_task.py
 dkist_processing_core/tests/test_workflow.py
 dkist_processing_core/tests/invalid_workflow_cyclic/__init__.py

{dkist_processing_core-7.2.1rc1 → dkist_processing_core-7.2.2rc2}/dkist_processing_core.egg-info/requires.txt

@@ -3,6 +3,8 @@ requests>=2.23
 talus<2.0,>=1.3.4
 pendulum
 nbformat>=5.9.2
+notebook<8.0,>=7.5.5
+nbconvert<8.0,>=7.16.6
 dkist-service-configuration<5.0,>=4.3.0
 pydantic>2.0
 

{dkist_processing_core-7.2.1rc1 → dkist_processing_core-7.2.2rc2}/pyproject.toml

@@ -26,6 +26,8 @@ dependencies = [
   "talus >= 1.3.4, <2.0",
   "pendulum",
   "nbformat >= 5.9.2",
+  "notebook >= 7.5.5, < 8.0",
+  "nbconvert >= 7.16.6, < 8.0",
   "dkist-service-configuration >=4.3.0, <5.0",
   "pydantic > 2.0",
 ]
@@ -63,6 +65,10 @@ markers = [
   "development: For tests that can only be run while developing with a sidecar proxy (as opposed to in bitbucket pipelines)",
   "long: For tests that take a long time to run",
 ]
+filterwarnings = [
+    # Turn all warnings into errors so they do not pass silently.
+    "error",
+]
 
 [tool.coverage.run]
 omit = [

dkist_processing_core-7.2.1rc1/changelog/75.misc.rst

@@ -1 +0,0 @@
-Upgrade Read the Docs LTS build image to Ubuntu 24.04.