dkist-processing-common 10.6.1rc5__tar.gz → 10.6.3__tar.gz

{dkist_processing_common-10.6.1rc5 → dkist_processing_common-10.6.3}/CHANGELOG.rst

@@ -1,3 +1,41 @@
+v10.6.3 (2025-03-27)
+====================
+
+Bugfixes
+--------
+
+- Fix a bug where transfer input dataset fails if any of the input dataset part documents are missing. (`#241 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/241>`__)
+- Pydantic model validator now returns the validated instance. (`#240 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/240>`__)
+
+v10.6.2 (2025-03-19)
+====================
+
+Bugfixes
+--------
+
+- Fix a bug in manual processing where a metadata store dataclass has been previously converted to
+  a Pydantic BaseModel.  Add a test that will catch future similar bugs. (`#239 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/239>`__)
+
+
+Misc
+----
+
+- Add coverage badge to README.rst. (`#238 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/238>`__)
+
+
+v10.6.1 (2025-03-14)
+====================
+
+Misc
+----
+
+- Convert dataclasses in the graphql model to Pydantic BaseModels for additional validation. In the
+  RecipeRunResponse class, configuration is converted from a JSON dictionary to its own Pydantic BaseModel.
+  In the InputDatasetPartResponse class, the inputDatasetPartDocument is now returned as a list of dictionaries. (`#236 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/236>`__)
+- Change returns from the metadata store queries into Pydantic BaseModel instances.  Remove unnecessary parsing
+  and error checking in the metadata store mixin. (`#236 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/236>`__)
+
+
 v10.6.0 (2025-03-03)
 ====================
 

{dkist_processing_common-10.6.1rc5 → dkist_processing_common-10.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 10.6.1rc5
+Version: 10.6.3
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -65,6 +65,8 @@ Requires-Dist: dkist-quality<2.0,>=1.2.1; extra == "quality"
 dkist-processing-common
 =======================
 
+|codecov|
+
 This repository works in concert with `dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_ and `dkist-processing-*instrument*` to
 form the DKIST calibration processing stack.
 
@@ -173,3 +175,6 @@ then the Bitbucket pipeline will fail. To be able to use the same tag you must d
     # Re-tag with the same version
     git tag vWHATEVER.THE.VERSION
     git push --tags origin main
+
+.. |codecov| image:: https://codecov.io/bb/dkistdc/dkist-processing-common/graph/badge.svg?token=3QSLGSEF3O
+ :target: https://codecov.io/bb/dkistdc/dkist-processing-common

{dkist_processing_common-10.6.1rc5 → dkist_processing_common-10.6.3}/README.rst

@@ -1,6 +1,8 @@
 dkist-processing-common
 =======================
 
+|codecov|
+
 This repository works in concert with `dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_ and `dkist-processing-*instrument*` to
 form the DKIST calibration processing stack.
 
@@ -109,3 +111,6 @@ then the Bitbucket pipeline will fail. To be able to use the same tag you must d
     # Re-tag with the same version
     git tag vWHATEVER.THE.VERSION
     git push --tags origin main
+
+.. |codecov| image:: https://codecov.io/bb/dkistdc/dkist-processing-common/graph/badge.svg?token=3QSLGSEF3O
+ :target: https://codecov.io/bb/dkistdc/dkist-processing-common

{dkist_processing_common-10.6.1rc5 → dkist_processing_common-10.6.3}/bitbucket-pipelines.yml

@@ -18,24 +18,28 @@ definitions:
           - chmod 755 ci_scripts/latest/execute_script.sh
         artifacts:
           - ci_scripts/latest/execute_script.sh
+
     - step: &lint
         caches:
           - pip
         name: Lint
         script:
           - ./ci_scripts/latest/execute_script.sh lint_python.sh
+
     - step: &changelog
         caches:
           - pip
         name: Changelog
         script:
           - ./ci_scripts/latest/execute_script.sh check_changelog.sh
+
     - step: &scan
         caches:
           - pip
         name: Scan
         script:
           - ./ci_scripts/latest/execute_script.sh scan_python.sh
+
     - step: &test
         caches:
           - pip
@@ -43,7 +47,9 @@ definitions:
         script:
           - pip install -U pip
           - pip install .[test]
-          - pytest -v -n auto -m "not development" --cov dkist_processing_common
+          - pytest -v -n auto -m "not development" --cov dkist_processing_common --cov-report=xml --cov-report=term-missing
+          - ./ci_scripts/latest/execute_script.sh upload_coverage.sh
+
         services:
           - redis
           - broker

{dkist_processing_common-10.6.1rc5 → dkist_processing_common-10.6.3}/dkist_processing_common/manual.py

@@ -182,7 +182,7 @@ def writing_metadata_store_record_provenance(self, is_task_manual: bool, library
         workflowVersion=self.workflow_version,
     )
     self.write(
-        data=asdict(params),
+        data=params.model_dump(),
         encoder=json_encoder,
         tags=["PROVENANCE_RECORD"],
         relative_path=f"{self.task_name}_provenance.json",

{dkist_processing_common-10.6.1rc5 → dkist_processing_common-10.6.3}/dkist_processing_common/models/wavelength.py

@@ -26,3 +26,4 @@ class WavelengthRange(BaseModel):
         """Validate that the max wavelength is greater than the min wavelength."""
         if self.min > self.max:
             raise ValueError("min is greater than max.  Values may be reversed.")
+        return self

{dkist_processing_common-10.6.1rc5 → dkist_processing_common-10.6.3}/dkist_processing_common/tasks/transfer_input_data.py

@@ -20,12 +20,17 @@ class TransferL0Data(WorkflowTaskBase, GlobusMixin, InputDatasetMixin):
 
     def download_input_dataset(self):
         """Get the input dataset document parts and save it to scratch with the appropriate tags."""
-        if doc := self.metadata_store_input_dataset_observe_frames.inputDatasetPartDocument:
-            self.write(doc, tags=Tag.input_dataset_observe_frames(), encoder=json_encoder)
-        if doc := self.metadata_store_input_dataset_calibration_frames.inputDatasetPartDocument:
-            self.write(doc, tags=Tag.input_dataset_calibration_frames(), encoder=json_encoder)
-        if doc := self.metadata_store_input_dataset_parameters.inputDatasetPartDocument:
-            self.write(doc, tags=Tag.input_dataset_parameters(), encoder=json_encoder)
+        if observe_frames := self.metadata_store_input_dataset_observe_frames:
+            observe_doc = observe_frames.inputDatasetPartDocument
+            self.write(observe_doc, tags=Tag.input_dataset_observe_frames(), encoder=json_encoder)
+        if calibration_frames := self.metadata_store_input_dataset_calibration_frames:
+            calibration_doc = calibration_frames.inputDatasetPartDocument
+            self.write(
+                calibration_doc, tags=Tag.input_dataset_calibration_frames(), encoder=json_encoder
+            )
+        if parameters := self.metadata_store_input_dataset_parameters:
+            parameters_doc = parameters.inputDatasetPartDocument
+            self.write(parameters_doc, tags=Tag.input_dataset_parameters(), encoder=json_encoder)
 
     def format_transfer_items(
         self, input_dataset_objects: list[InputDatasetObject]

dkist_processing_common-10.6.3/dkist_processing_common/tests/test_manual_processing.py

@@ -0,0 +1,34 @@
+import pytest
+
+from dkist_processing_common.manual import ManualProcessing
+from dkist_processing_common.tasks import WorkflowTaskBase
+
+
+class ProvenanceTask(WorkflowTaskBase):
+    record_provenance = True
+
+    def run(self):
+        ...
+
+
+@pytest.fixture(scope="function")
+def manual_processing_run(tmp_path, recipe_run_id):
+    with ManualProcessing(
+        recipe_run_id=recipe_run_id,
+        workflow_path=tmp_path,
+        workflow_name="manual",
+        workflow_version="manual",
+    ) as manual_processing_run:
+        yield manual_processing_run
+
+
+def test_manual_record_provenance(tmp_path, recipe_run_id, manual_processing_run):
+    """
+    Given: A WorkflowTaskBase subclass with provenance recording turned on
+    When: Running the task with the ManualProcessing wrapper
+    Then: The provenance record exists on disk
+    """
+    manual_processing_run.run_task(task=ProvenanceTask)
+    directory = tmp_path / str(recipe_run_id)
+    provenance_file = directory / "ProvenanceTask_provenance.json"
+    assert provenance_file.exists()

{dkist_processing_common-10.6.1rc5 → dkist_processing_common-10.6.3}/dkist_processing_common/tests/test_transfer_input_data.py

@@ -17,10 +17,20 @@ class TransferL0DataTask(TransferL0Data):
         ...
 
 
-@pytest.fixture
-def transfer_l0_data_task(recipe_run_id, tmp_path, mocker):
+class FakeGQLClientMissingPart(FakeGQLClient):
+    """Same metadata mocker with calibration input dataset part missing."""
+
+    def execute_gql_query(self, **kwargs):
+        original_response = super().execute_gql_query(**kwargs)
+        # Remove calibration frames part
+        del original_response[0].recipeInstance.inputDataset.inputDatasetInputDatasetParts[2]
+        return original_response
+
+
+def _transfer_l0_data_task_with_client(recipe_run_id, tmp_path, mocker, client_cls):
     mocker.patch(
-        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
+        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
+        new=client_cls,
     )
     with TransferL0DataTask(
         recipe_run_id=recipe_run_id,
@@ -35,6 +45,18 @@ def transfer_l0_data_task(recipe_run_id, tmp_path, mocker):
         task._purge()
 
 
+@pytest.fixture
+def transfer_l0_data_task(recipe_run_id, tmp_path, mocker):
+    yield from _transfer_l0_data_task_with_client(recipe_run_id, tmp_path, mocker, FakeGQLClient)
+
+
+@pytest.fixture
+def transfer_l0_data_task_missing_part(recipe_run_id, tmp_path, mocker):
+    yield from _transfer_l0_data_task_with_client(
+        recipe_run_id, tmp_path, mocker, FakeGQLClientMissingPart
+    )
+
+
 def test_download_dataset(transfer_l0_data_task):
     """
     :Given: a TransferL0Data task with a valid input dataset
@@ -63,6 +85,29 @@ def test_download_dataset(transfer_l0_data_task):
     assert parameters_doc_from_file == expected_parameters_doc
 
 
+def test_download_dataset_missing_part(transfer_l0_data_task_missing_part):
+    """
+    :Given: a TransferL0Data task with a valid input dataset without calibration frames
+    :When: downloading the dataset documents from the metadata store
+    :Then: the correct number of documents are written to disk
+    """
+    # Given
+    task = transfer_l0_data_task_missing_part
+    # When
+    task.download_input_dataset()
+    # Then
+    observe_doc_from_file = next(
+        task.read(tags=Tag.input_dataset_observe_frames(), decoder=json_decoder)
+    )
+    parameters_doc_from_file = next(
+        task.read(tags=Tag.input_dataset_parameters(), decoder=json_decoder)
+    )
+    with pytest.raises(StopIteration):
+        calibration_doc_from_file = next(
+            task.read(tags=Tag.input_dataset_calibration_frames(), decoder=json_decoder)
+        )
+
+
 def test_format_frame_transfer_items(transfer_l0_data_task):
     """
     :Given: a TransferL0Data task with a downloaded input dataset

{dkist_processing_common-10.6.1rc5 → dkist_processing_common-10.6.3}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 10.6.1rc5
+Version: 10.6.3
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -65,6 +65,8 @@ Requires-Dist: dkist-quality<2.0,>=1.2.1; extra == "quality"
 dkist-processing-common
 =======================
 
+|codecov|
+
 This repository works in concert with `dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_ and `dkist-processing-*instrument*` to
 form the DKIST calibration processing stack.
 
@@ -173,3 +175,6 @@ then the Bitbucket pipeline will fail. To be able to use the same tag you must d
     # Re-tag with the same version
     git tag vWHATEVER.THE.VERSION
     git push --tags origin main
+
+.. |codecov| image:: https://codecov.io/bb/dkistdc/dkist-processing-common/graph/badge.svg?token=3QSLGSEF3O
+ :target: https://codecov.io/bb/dkistdc/dkist-processing-common

{dkist_processing_common-10.6.1rc5 → dkist_processing_common-10.6.3}/dkist_processing_common.egg-info/SOURCES.txt

@@ -7,8 +7,6 @@ README.rst
 bitbucket-pipelines.yml
 pyproject.toml
 changelog/.gitempty
-changelog/236.misc.1.rst
-changelog/236.misc.rst
 dkist_processing_common/__init__.py
 dkist_processing_common/config.py
 dkist_processing_common/manual.py
@@ -96,6 +94,7 @@ dkist_processing_common/tests/test_flower_pot.py
 dkist_processing_common/tests/test_input_dataset.py
 dkist_processing_common/tests/test_interservice_bus.py
 dkist_processing_common/tests/test_interservice_bus_mixin.py
+dkist_processing_common/tests/test_manual_processing.py
 dkist_processing_common/tests/test_output_data_base.py
 dkist_processing_common/tests/test_parameters.py
 dkist_processing_common/tests/test_parse_l0_input_data.py

dkist_processing_common-10.6.1rc5/changelog/236.misc.1.rst

@@ -1,2 +0,0 @@
-Change returns from the metadata store queries into Pydantic BaseModel instances.  Remove unnecessary parsing
-and error checking in the metadata store mixin.

dkist_processing_common-10.6.1rc5/changelog/236.misc.rst

@@ -1,3 +0,0 @@
-Convert dataclasses in the graphql model to Pydantic BaseModels for additional validation. In the
-RecipeRunResponse class, configuration is converted from a JSON dictionary to its own Pydantic BaseModel.
-In the InputDatasetPartResponse class, the inputDatasetPartDocument is now returned as a list of dictionaries.