dkist-processing-core 7.2.2rc1__tar.gz → 7.2.2rc2__tar.gz

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.2.2rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-core
-Version: 7.2.2rc1
+Version: 7.2.2rc2
 Summary: Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.2.2rc2}/dkist_processing_core/build_utils.py

@@ -119,10 +119,7 @@ class NotebookDockerfile:
             "COPY . /app",
             "WORKDIR /app",
             "RUN python -m pip install -U pip",
-            "RUN pip install notebook",
-            "RUN pip freeze | grep notebook= > constraints.txt",
-            "RUN cat constraints.txt",
-            "RUN python -m pip install -c constraints.txt .",
+            "RUN python -m pip install .",
         ]
 
     @property

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.2.2rc2}/dkist_processing_core/notebook_utils.py

@@ -2,19 +2,18 @@
 
 import time
 from pathlib import Path
+from uuid import uuid4
 
 import nbformat
 from nbconvert import MarkdownExporter
 
-DEFAULT_EXPORT_SENTINEL = "MPW_EXPORT_READY"
-
 
 def wait_for_notebook_save(
-    notebook_path: Path | str,
-    sentinel: str = DEFAULT_EXPORT_SENTINEL,
-    attempts=120,
-    delay_s=1,
-):
+    notebook_path: Path,
+    sentinel: str | None = None,
+    attempts: int = 120,
+    delay_s: int = 1,
+) -> None:
     """
     Wait until the notebook on disk contains the sentinel output.
 
@@ -37,28 +36,20 @@ def wait_for_notebook_save(
     -------
     None if the sentinel is found within the notebook outputs, otherwise raises a TimeoutError after the specified number of attempts.
     """
+    sentinel = sentinel or uuid4().hex
+
     print(f"Waiting for notebook to save with sentinel '{sentinel}'...", flush=True)
-    for attempt in range(attempts):
-        with open(notebook_path, "r", encoding="utf-8") as f:
-            nb = nbformat.read(f, as_version=nbformat.NO_CONVERT)
-
-        # Search outputs
-        for cell in nb.cells:
-            if cell.cell_type == "code":
-                for output in cell.get("outputs", []):
-                    text = ""
-
-                    if output.output_type == "stream":
-                        text = output.get("text", "")
-                    elif output.output_type in ("execute_result", "display_data"):
-                        text = str(output.get("data", ""))
-                    if sentinel in text:
-                        return
+
+    for _ in range(attempts):
+        text = notebook_path.read_text(encoding="utf-8")
+        if sentinel in text:
+            return
         time.sleep(delay_s)
+
     raise TimeoutError("Notebook did not save updated outputs in time")
 
 
-def export_notebook_by_path(notebook_path) -> bytes:
+def export_notebook_by_path(notebook_path: Path) -> bytes:
     """
     Export the notebook at the given path to markdown format and return the resulting bytes.
 
@@ -66,10 +57,14 @@ def export_notebook_by_path(notebook_path) -> bytes:
     ----------
     notebook_path
         The path to the notebook file to export.
+
+    Returns
+    -------
+    The exported notebook in markdown format as bytes.
     """
-    with open(notebook_path, "r", encoding="utf-8") as f:
-        nb = nbformat.read(f, as_version=4)
+    notebook_contents = notebook_path.read_text(encoding="utf-8")
+    notebook = nbformat.reads(notebook_contents, as_version=nbformat.NO_CONVERT)
 
     exporter = MarkdownExporter()
-    body, _ = exporter.from_notebook_node(nb)
+    body, _ = exporter.from_notebook_node(notebook)
     return body.encode("utf-8")

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.2.2rc2}/dkist_processing_core/tests/test_notebook_utils.py

@@ -1,6 +1,7 @@
 """Tests for the notebook_utils module"""
 
 from pathlib import Path
+from uuid import uuid4
 
 import nbformat
 import pytest
@@ -9,7 +10,6 @@ from nbformat.v4 import new_markdown_cell
 from nbformat.v4 import new_notebook
 from nbformat.v4 import new_output
 
-from dkist_processing_core.notebook_utils import DEFAULT_EXPORT_SENTINEL
 from dkist_processing_core.notebook_utils import export_notebook_by_path
 from dkist_processing_core.notebook_utils import wait_for_notebook_save
 
@@ -28,7 +28,12 @@ def notebook_path_factory(tmp_path: Path):
 
 
 @pytest.fixture
-def notebook_with_sentinel_output(notebook_path_factory) -> Path:
+def notebook_sentinel() -> str:
+    return uuid4().hex
+
+
+@pytest.fixture
+def notebook_with_sentinel_output(notebook_path_factory, notebook_sentinel) -> Path:
     return notebook_path_factory(
         cells=[
             new_markdown_cell("# Title"),
@@ -38,7 +43,7 @@ def notebook_with_sentinel_output(notebook_path_factory) -> Path:
                     new_output(
                         output_type="stream",
                         name="stdout",
-                        text=f"before {DEFAULT_EXPORT_SENTINEL} after",
+                        text=f"before {notebook_sentinel} after",
                     )
                 ],
             ),
@@ -65,10 +70,12 @@ def notebook_without_sentinel_output(notebook_path_factory) -> Path:
 
 
 def test_wait_for_notebook_save_succeeds_when_sentinel_in_code_cell_output(
-    notebook_with_sentinel_output,
+    notebook_with_sentinel_output, notebook_sentinel
 ):
     # no error raised
-    wait_for_notebook_save(notebook_with_sentinel_output, attempts=1, delay_s=0)
+    wait_for_notebook_save(
+        notebook_with_sentinel_output, sentinel=notebook_sentinel, attempts=1, delay_s=0
+    )
 
 
 def test_wait_for_notebook_save_times_out_when_sentinel_missing(
@@ -78,7 +85,9 @@ def test_wait_for_notebook_save_times_out_when_sentinel_missing(
         wait_for_notebook_save(notebook_without_sentinel_output, attempts=1, delay_s=0)
 
 
-def test_export_notebook_by_path_returns_markdown_bytes(notebook_with_sentinel_output) -> None:
+def test_export_notebook_by_path_returns_markdown_bytes(
+    notebook_with_sentinel_output, notebook_sentinel
+) -> None:
     exported = export_notebook_by_path(notebook_with_sentinel_output)
 
     assert isinstance(exported, bytes)
@@ -87,4 +96,4 @@ def test_export_notebook_by_path_returns_markdown_bytes(notebook_with_sentinel_o
 
     assert "# Title" in output  # markdown
     assert "print('ready')" in output  # code
-    assert DEFAULT_EXPORT_SENTINEL in output  # output
+    assert notebook_sentinel in output  # output

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.2.2rc2}/dkist_processing_core/workflow.py

@@ -13,7 +13,7 @@ from dkist_processing_core.node import task_type_hint
 from dkist_processing_core.node import upstreams_type_hint
 from dkist_processing_core.resource_queue import ResourceQueue
 
-__all__ = ["Workflow", "workflow_name_from_details"]
+__all__ = ["Workflow", "workflow_name_from_details", "dag_name"]
 
 MAXIMUM_ALLOWED_WORKFLOW_NAME_LENGTH = 100
 
@@ -226,7 +226,7 @@ class Workflow:
             f.write("\n")
         return workflow_py
 
-    def export_notebook(self, path: str | Path | None = None):
+    def export_notebook(self, path: str | Path | None = None) -> Path:
         """Render the workflow as a jupyter notebook."""
         path = path or "notebooks/"
         path = Path(path)
@@ -241,7 +241,7 @@ class Workflow:
             nbf.write(nb, f)
         return notebook_ipynb
 
-    def topological_sort(self) -> [Node]:
+    def topological_sort(self) -> list[Node]:
         """Use a topological sort to find a valid linear order for task execution."""
         node_task_names = {node.task.__name__: node for node in self.nodes}
         node_upstream_tasks = {node.task: node.upstreams for node in self.nodes}

{dkist_processing_core-7.2.2rc1 → dkist_processing_core-7.2.2rc2}/dkist_processing_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-core
-Version: 7.2.2rc1
+Version: 7.2.2rc2
 Summary: Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause