dkist-processing-core 7.2.1__tar.gz → 7.2.2rc1__tar.gz

{dkist_processing_core-7.2.1 → dkist_processing_core-7.2.2rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-core
-Version: 7.2.1
+Version: 7.2.2rc1
 Summary: Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -18,6 +18,8 @@ Requires-Dist: requests>=2.23
 Requires-Dist: talus<2.0,>=1.3.4
 Requires-Dist: pendulum
 Requires-Dist: nbformat>=5.9.2
+Requires-Dist: notebook<8.0,>=7.5.5
+Requires-Dist: nbconvert<8.0,>=7.16.6
 Requires-Dist: dkist-service-configuration<5.0,>=4.3.0
 Requires-Dist: pydantic>2.0
 Provides-Extra: test

dkist_processing_core-7.2.2rc1/changelog/72.misc.rst

@@ -0,0 +1 @@
+Add an environment variable to indicate when the execution environment is a Jupyter notebook.

dkist_processing_core-7.2.2rc1/changelog/76.feature.1.rst

@@ -0,0 +1 @@
+Add utility for working with the notebook build target which supports waiting for a notebook to autosave.

dkist_processing_core-7.2.2rc1/changelog/76.feature.2.rst

@@ -0,0 +1 @@
+Add utility for working with the notebook build target which supports exporting a notebook as markdown.

dkist_processing_core-7.2.2rc1/changelog/77.misc.rst

@@ -0,0 +1 @@
+Refactor the dag naming function to be accessible outside of the context of a Workflow instance.

{dkist_processing_core-7.2.1 → dkist_processing_core-7.2.2rc1}/dkist_processing_core/build_utils.py

@@ -115,6 +115,7 @@ class NotebookDockerfile:
     def setup(self) -> list[str]:
         """Environment setup lines."""
         return [
+            "ENV IS_NOTEBOOK_EXECUTION_ENVIRONMENT=true",
             "COPY . /app",
             "WORKDIR /app",
             "RUN python -m pip install -U pip",

{dkist_processing_core-7.2.1 → dkist_processing_core-7.2.2rc1}/dkist_processing_core/config.py

@@ -27,6 +27,11 @@ class DKISTProcessingCoreConfiguration(InstrumentedMeshServiceConfigurationBase)
     )
     pip_timeout: int = Field(default=15, description="Timeout for pip installs in seconds.")
     pip_retries: int = Field(default=5, description="Number of retries for pip installs.")
+    is_notebook_execution_environment: bool = Field(
+        default=False,
+        description="Indication of whether the code is running in a notebook execution environment.",
+        examples=[True, False],
+    )
 
     @property
     def isb_mesh_service(self) -> MeshService:

dkist_processing_core-7.2.2rc1/dkist_processing_core/notebook_utils.py

@@ -0,0 +1,75 @@
+"""Utilities for working with the notebook execution environment."""
+
+import time
+from pathlib import Path
+
+import nbformat
+from nbconvert import MarkdownExporter
+
+DEFAULT_EXPORT_SENTINEL = "MPW_EXPORT_READY"
+
+
+def wait_for_notebook_save(
+    notebook_path: Path | str,
+    sentinel: str = DEFAULT_EXPORT_SENTINEL,
+    attempts=120,
+    delay_s=1,
+):
+    """
+    Wait until the notebook on disk contains the sentinel output.
+
+    Parameters
+    ----------
+    notebook_path
+        The path to the notebook file to check for the sentinel output.
+
+    sentinel
+        The string to search for in the notebook outputs to confirm that the notebook has been saved with the expected output.
+
+    attempts
+        The number of times to check the notebook for the sentinel output before giving up and raising a
+        TimeoutError.
+
+    delay_s
+        The number of seconds to wait between attempts to check the notebook for the sentinel output.
+
+    Returns
+    -------
+    None if the sentinel is found within the notebook outputs, otherwise raises a TimeoutError after the specified number of attempts.
+    """
+    print(f"Waiting for notebook to save with sentinel '{sentinel}'...", flush=True)
+    for attempt in range(attempts):
+        with open(notebook_path, "r", encoding="utf-8") as f:
+            nb = nbformat.read(f, as_version=nbformat.NO_CONVERT)
+
+        # Search outputs
+        for cell in nb.cells:
+            if cell.cell_type == "code":
+                for output in cell.get("outputs", []):
+                    text = ""
+
+                    if output.output_type == "stream":
+                        text = output.get("text", "")
+                    elif output.output_type in ("execute_result", "display_data"):
+                        text = str(output.get("data", ""))
+                    if sentinel in text:
+                        return
+        time.sleep(delay_s)
+    raise TimeoutError("Notebook did not save updated outputs in time")
+
+
+def export_notebook_by_path(notebook_path) -> bytes:
+    """
+    Export the notebook at the given path to markdown format and return the resulting bytes.
+
+    Parameters
+    ----------
+    notebook_path
+        The path to the notebook file to export.
+    """
+    with open(notebook_path, "r", encoding="utf-8") as f:
+        nb = nbformat.read(f, as_version=4)
+
+    exporter = MarkdownExporter()
+    body, _ = exporter.from_notebook_node(nb)
+    return body.encode("utf-8")

dkist_processing_core-7.2.2rc1/dkist_processing_core/tests/test_notebook_utils.py

@@ -0,0 +1,90 @@
+"""Tests for the notebook_utils module"""
+
+from pathlib import Path
+
+import nbformat
+import pytest
+from nbformat.v4 import new_code_cell
+from nbformat.v4 import new_markdown_cell
+from nbformat.v4 import new_notebook
+from nbformat.v4 import new_output
+
+from dkist_processing_core.notebook_utils import DEFAULT_EXPORT_SENTINEL
+from dkist_processing_core.notebook_utils import export_notebook_by_path
+from dkist_processing_core.notebook_utils import wait_for_notebook_save
+
+
+@pytest.fixture
+def notebook_path_factory(tmp_path: Path):
+    def _make_notebook(*, cells=None) -> Path:
+        notebook_path = tmp_path / "test.ipynb"
+        cells = cells or []
+        nb = new_notebook(cells=cells)
+        with notebook_path.open("w", encoding="utf-8") as f:
+            nbformat.write(nb, f)
+        return notebook_path
+
+    return _make_notebook
+
+
+@pytest.fixture
+def notebook_with_sentinel_output(notebook_path_factory) -> Path:
+    return notebook_path_factory(
+        cells=[
+            new_markdown_cell("# Title"),
+            new_code_cell(
+                source="print('ready')",
+                outputs=[
+                    new_output(
+                        output_type="stream",
+                        name="stdout",
+                        text=f"before {DEFAULT_EXPORT_SENTINEL} after",
+                    )
+                ],
+            ),
+        ]
+    )
+
+
+@pytest.fixture
+def notebook_without_sentinel_output(notebook_path_factory) -> Path:
+    return notebook_path_factory(
+        cells=[
+            new_code_cell(
+                source="print('not ready')",
+                outputs=[
+                    new_output(
+                        output_type="stream",
+                        name="stdout",
+                        text="some other output",
+                    )
+                ],
+            )
+        ]
+    )
+
+
+def test_wait_for_notebook_save_succeeds_when_sentinel_in_code_cell_output(
+    notebook_with_sentinel_output,
+):
+    # no error raised
+    wait_for_notebook_save(notebook_with_sentinel_output, attempts=1, delay_s=0)
+
+
+def test_wait_for_notebook_save_times_out_when_sentinel_missing(
+    notebook_without_sentinel_output,
+):
+    with pytest.raises(TimeoutError, match="Notebook did not save updated outputs in time"):
+        wait_for_notebook_save(notebook_without_sentinel_output, attempts=1, delay_s=0)
+
+
+def test_export_notebook_by_path_returns_markdown_bytes(notebook_with_sentinel_output) -> None:
+    exported = export_notebook_by_path(notebook_with_sentinel_output)
+
+    assert isinstance(exported, bytes)
+
+    output = exported.decode("utf-8")
+
+    assert "# Title" in output  # markdown
+    assert "print('ready')" in output  # code
+    assert DEFAULT_EXPORT_SENTINEL in output  # output

{dkist_processing_core-7.2.1 → dkist_processing_core-7.2.2rc1}/dkist_processing_core/tests/test_workflow.py

@@ -9,6 +9,7 @@ from airflow import DAG
 from dkist_processing_core import ResourceQueue
 from dkist_processing_core import Workflow
 from dkist_processing_core.workflow import MAXIMUM_ALLOWED_WORKFLOW_NAME_LENGTH
+from dkist_processing_core.workflow import _check_dag_name_characters
 from dkist_processing_core.workflow import workflow_name_from_details
 
 
@@ -165,9 +166,9 @@ def test_check_dag_name_characters():
     When: checking if it is a valid airflow name or not
     Then: correctly identify valid and invalid names
     """
-    Workflow.check_dag_name_characters(dag_name="This_dag_name_is_valid")
+    _check_dag_name_characters(dag_name="This_dag_name_is_valid")
     with pytest.raises(ValueError):
-        Workflow.check_dag_name_characters(dag_name="Invalid*dag*name")
+        _check_dag_name_characters(dag_name="Invalid*dag*name")
 
 
 @pytest.mark.parametrize(

{dkist_processing_core-7.2.1 → dkist_processing_core-7.2.2rc1}/dkist_processing_core/workflow.py

@@ -34,6 +34,35 @@ def workflow_name_from_details(
     return workflow_name
 
 
+def _check_dag_name_characters(dag_name: str):
+    """
+    Figure out if the dag name is an Airflow-allowed name.
+
+    Can only contain
+    * ascii letters
+    * numbers
+    * dash (-)
+    * dot (.)
+    * underscore (_)
+
+    Raise error if non-allowed characters are found.
+    """
+    allowed_chars = [c for c in string.ascii_letters] + ["-", ".", "_"] + [n for n in string.digits]
+    if not all([char in allowed_chars for char in dag_name]):
+        raise ValueError(
+            f"Dag name {dag_name} contains invalid characters. "
+            f"Only ascii letters and the dash, dot, and "
+            f"underscore symbols are permitted."
+        )
+
+
+def dag_name(workflow_name: str, workflow_version: str) -> str:
+    """Return the dag name created from its constituent parts."""
+    result = f"{workflow_name}_{workflow_version}"
+    _check_dag_name_characters(result)  # raise an error if in valid
+    return result
+
+
 class Workflow:
     """
     Abstraction to create a workflow in 1 or more target execution environment.
@@ -116,33 +145,7 @@ class Workflow:
     @property
     def dag_name(self) -> str:
         """Return the dag name created from its constituent parts."""
-        result = f"{self.workflow_name}_{self.workflow_version}"
-        self.check_dag_name_characters(result)  # raise an error if in valid
-        return result
-
-    @staticmethod
-    def check_dag_name_characters(dag_name: str):
-        """
-        Figure out if the dag name is an Airflow-allowed name.
-
-        Can only contain
-        * ascii letters
-        * numbers
-        * dash (-)
-        * dot (.)
-        * underscore (_)
-
-        Raise error if non-allowed characters are found.
-        """
-        allowed_chars = (
-            [c for c in string.ascii_letters] + ["-", ".", "_"] + [n for n in string.digits]
-        )
-        if not all([char in allowed_chars for char in dag_name]):
-            raise ValueError(
-                f"Dag name {dag_name} contains invalid characters. "
-                f"Only ascii letters and the dash, dot, and "
-                f"underscore symbols are permitted."
-            )
+        return dag_name(workflow_name=self.workflow_name, workflow_version=self.workflow_version)
 
     @property
     def dag_tags(self) -> str:

{dkist_processing_core-7.2.1 → dkist_processing_core-7.2.2rc1}/dkist_processing_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-core
-Version: 7.2.1
+Version: 7.2.2rc1
 Summary: Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -18,6 +18,8 @@ Requires-Dist: requests>=2.23
 Requires-Dist: talus<2.0,>=1.3.4
 Requires-Dist: pendulum
 Requires-Dist: nbformat>=5.9.2
+Requires-Dist: notebook<8.0,>=7.5.5
+Requires-Dist: nbconvert<8.0,>=7.16.6
 Requires-Dist: dkist-service-configuration<5.0,>=4.3.0
 Requires-Dist: pydantic>2.0
 Provides-Extra: test

{dkist_processing_core-7.2.1 → dkist_processing_core-7.2.2rc1}/dkist_processing_core.egg-info/SOURCES.txt

@@ -7,11 +7,16 @@ README.rst
 bitbucket-pipelines.yml
 pyproject.toml
 changelog/.gitempty
+changelog/72.misc.rst
+changelog/76.feature.1.rst
+changelog/76.feature.2.rst
+changelog/77.misc.rst
 dkist_processing_core/__init__.py
 dkist_processing_core/build_utils.py
 dkist_processing_core/config.py
 dkist_processing_core/failure_callback.py
 dkist_processing_core/node.py
+dkist_processing_core/notebook_utils.py
 dkist_processing_core/resource_queue.py
 dkist_processing_core/task.py
 dkist_processing_core/workflow.py
@@ -27,6 +32,7 @@ dkist_processing_core/tests/test_build_utils.py
 dkist_processing_core/tests/test_export.py
 dkist_processing_core/tests/test_failure_callback.py
 dkist_processing_core/tests/test_node.py
+dkist_processing_core/tests/test_notebook_utils.py
 dkist_processing_core/tests/test_task.py
 dkist_processing_core/tests/test_workflow.py
 dkist_processing_core/tests/invalid_workflow_cyclic/__init__.py

{dkist_processing_core-7.2.1 → dkist_processing_core-7.2.2rc1}/dkist_processing_core.egg-info/requires.txt

@@ -3,6 +3,8 @@ requests>=2.23
 talus<2.0,>=1.3.4
 pendulum
 nbformat>=5.9.2
+notebook<8.0,>=7.5.5
+nbconvert<8.0,>=7.16.6
 dkist-service-configuration<5.0,>=4.3.0
 pydantic>2.0
 

{dkist_processing_core-7.2.1 → dkist_processing_core-7.2.2rc1}/pyproject.toml

@@ -26,6 +26,8 @@ dependencies = [
   "talus >= 1.3.4, <2.0",
   "pendulum",
   "nbformat >= 5.9.2",
+  "notebook >= 7.5.5, < 8.0",
+  "nbconvert >= 7.16.6, < 8.0",
   "dkist-service-configuration >=4.3.0, <5.0",
   "pydantic > 2.0",
 ]