dkist-processing-common 13.0.5rc2__tar.gz → 13.0.6__tar.gz

{dkist_processing_common-13.0.5rc2 → dkist_processing_common-13.0.6}/CHANGELOG.rst

@@ -1,3 +1,24 @@
+v13.0.6 (2026-04-21)
+====================
+
+Misc
+----
+
+- Implement updated gqlclient v2.0.0, which uses Pydantic v2 BaseModels to
+  validate graphql parameters and responses. (`#283 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/283>`__)
+- Raise an informative error if attempting to read or write files in tasks with tags=None.  Tags are now a required
+  argument for writing with the WorkflowFileSystem.  Read and write with an empty list for tags is still supported. (`#327 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/327>`__)
+
+
+v13.0.5 (2026-04-20)
+====================
+
+Features
+--------
+
+- Remove 4 unused quality metric codes: RANGE, NOISE, SENSITIVITY, HISTORICAL. (`#329 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/329>`__)
+
+
 v13.0.4 (2026-04-16)
 ====================
 

{dkist_processing_common-13.0.5rc2 → dkist_processing_common-13.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 13.0.5rc2
+Version: 13.0.6
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -23,7 +23,7 @@ Requires-Dist: dkist-service-configuration<5.0,>=4.3.0
 Requires-Dist: dkist-spectral-lines<4.0,>=3.0.0
 Requires-Dist: solar-wavelength-calibration<3.0,>=2.0.0
 Requires-Dist: globus-sdk<5.0.0,>=4.0.0
-Requires-Dist: gqlclient[pydantic]==1.2.3
+Requires-Dist: gqlclient==2.0.0
 Requires-Dist: sqids==0.5.1
 Requires-Dist: matplotlib>=3.4
 Requires-Dist: moviepy>=2.0.0

{dkist_processing_common-13.0.5rc2 → dkist_processing_common-13.0.6}/dkist_processing_common/_util/graphql.py

@@ -5,9 +5,9 @@ from typing import Any
 from typing import Callable
 
 import requests
-from gqlclient.base import DefaultParameters
 from gqlclient.base import GraphQLClientBase
 from gqlclient.request_wrap import wrap_request
+from pydantic import BaseModel
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
 
@@ -59,14 +59,13 @@ class GraphQLClient(GraphQLClientBase):
     def execute_gql_query(
         self,
         query_base: str,
-        query_response_cls: type,
-        query_parameters: object | None = DefaultParameters,
-        response_encoder: Callable[[str, list[dict] | dict, type], Any] | None = None,
+        query_response_cls: type[BaseModel],
+        query_parameters: BaseModel | None = None,
+        response_encoder: Callable[[str, dict | list[dict], type[BaseModel]], Any] | None = None,
         **kwargs,
     ) -> Any:
         """Execute gql query with parameters dynamically wrapped."""
-        if query_parameters is not None and query_parameters is not DefaultParameters:
-            query_parameters = wrap_request(query_parameters)
+        query_parameters = wrap_request(query_parameters)
         return super().execute_gql_query(
             query_base=query_base,
             query_response_cls=query_response_cls,
@@ -78,9 +77,9 @@ class GraphQLClient(GraphQLClientBase):
     def execute_gql_mutation(
         self,
         mutation_base: str,
-        mutation_parameters: object,
-        mutation_response_cls: type | None = None,
-        response_encoder: Callable[[str, list[dict] | dict, type], Any] | None = None,
+        mutation_parameters: BaseModel,
+        mutation_response_cls: type[BaseModel],
+        response_encoder: Callable[[str, dict | list[dict], type[BaseModel]], Any] | None = None,
         **kwargs,
     ) -> Any:
         """Execute gql mutation with parameters dynamically wrapped."""

{dkist_processing_common-13.0.5rc2 → dkist_processing_common-13.0.6}/dkist_processing_common/_util/scratch.py

@@ -5,7 +5,9 @@ from contextlib import contextmanager
 from os import umask
 from pathlib import Path
 from shutil import rmtree
+from typing import Any
 from typing import Generator
+from typing import Iterable
 
 from dkist_processing_common._util.tags import TagDB
 from dkist_processing_common.config import common_configurations
@@ -82,19 +84,19 @@ class WorkflowFileSystem:
         return self.workflow_base_path / relative_path
 
     @staticmethod
-    def parse_tags(tags: str | list | None) -> list:
-        """Parse tags to support an individual tag in the form of a string or an arbitrarily nested list of strings."""
-        if tags is None:
-            return []
-        if isinstance(tags, str):
-            return [tags]
-        return _flatten_list(tags)
+    def parse_tags(tags: str | Iterable[str]) -> list:
+        """Parse tags to support an individual tag in the form of a string or an arbitrarily nested iterable of strings."""
+        if isinstance(tags, Iterable):
+            if isinstance(tags, str):
+                return [tags]
+            return _flatten_tags(tags)
+        raise TypeError(f"'tags' must be str or other iterator. Received {tags=}")
 
     def write(
         self,
         file_obj: bytes,
         relative_path: Path | str,
-        tags: str | list | None = None,
+        tags: str | Iterable[str],
         overwrite: bool = False,
     ) -> None:
         """
@@ -146,7 +148,7 @@ class WorkflowFileSystem:
         path.unlink(missing_ok=True)
         self._tag_db.clear_value(value=path)
 
-    def tag(self, path: Path | str, tags: list | str) -> None:
+    def tag(self, path: Path | str, tags: str | Iterable[str]) -> None:
         """
         Tag existing paths.
 
@@ -184,7 +186,7 @@ class WorkflowFileSystem:
             self._audit_new_tag_cache[tag] = tag_is_new
         return self._audit_new_tag_cache[tag]
 
-    def tags(self, path: Path | str):
+    def tags(self, path: Path | str) -> list[str]:
         """
         Return the tags associated with the given file object.
 
@@ -199,13 +201,13 @@ class WorkflowFileSystem:
         value = str(path)
         return self._tag_db.tags_for_value(value=value)
 
-    def remove_tags(self, path: Path | str, tags: list | str) -> None:
+    def remove_tags(self, path: Path | str, tags: str | Iterable[str]) -> None:
         """Remove a tag or tags from a given path."""
         tags = self.parse_tags(tags)
         for tag in tags:
             self._tag_db.remove(tag, str(path))
 
-    def find_any(self, tags: str | list) -> Generator[Path, None, None]:
+    def find_any(self, tags: str | Iterable[str]) -> Generator[Path, None, None]:
         """
         Return a generator of Path objects that are tagged by the union of the input tags.
 
@@ -224,7 +226,7 @@ class WorkflowFileSystem:
         for path in paths:
             yield Path(path)
 
-    def find_all(self, tags: str | list) -> Generator[Path, None, None]:
+    def find_all(self, tags: str | Iterable[str]) -> Generator[Path, None, None]:
         """
         Return a generator of Path objects that are tagged by the intersection of the input tags.
 
@@ -243,7 +245,7 @@ class WorkflowFileSystem:
         for path in paths:
             yield Path(path)
 
-    def count_any(self, tags: str | list) -> int:
+    def count_any(self, tags: str | Iterable[str]) -> int:
         """
         Return the number of objects that are tagged by the union of the input tags.
 
@@ -259,7 +261,7 @@ class WorkflowFileSystem:
         tags = self.parse_tags(tags)
         return len(self._tag_db.any(tags))
 
-    def count_all(self, tags: str | list) -> int:
+    def count_all(self, tags: str | Iterable[str]) -> int:
         """
         Return the number of objects that are tagged by the intersection of the input tags.
 
@@ -326,12 +328,12 @@ class WorkflowFileSystem:
         self.close()
 
 
-def _flatten_list(elements: list) -> list:
-    """Flatten an arbitrarily nested list."""
+def _flatten_tags(tags: Iterable[Any]) -> list[str]:
+    """Flatten an arbitrarily nested iterable into a flat list of strings."""
     result = []
-    for element in elements:
-        if isinstance(element, list):
-            result.extend(_flatten_list(element))
+    for tag in tags:
+        if isinstance(tag, str):
+            result.append(tag)
         else:
-            result.append(element)
+            result.extend(_flatten_tags(tag))
     return result

{dkist_processing_common-13.0.5rc2 → dkist_processing_common-13.0.6}/dkist_processing_common/_util/tags.py

@@ -52,7 +52,7 @@ class TagDB:
     def _format_query_result(result: list[bytes]) -> set[str]:
         return {r.decode("utf8") for r in result}
 
-    def _add_name_space(self, tags: Iterable[str] | str) -> Iterable[str] | str:
+    def _add_name_space(self, tags: Iterable[str] | str) -> list[str] | str:
         if isinstance(tags, str):
             return self.namespace + tags
         return [f"{self.namespace}{t}" for t in tags]
@@ -184,7 +184,7 @@ class TagDB:
         r = self.db.sinter(tags)
         return self._format_query_result(r)
 
-    def tags_for_value(self, value: str) -> Iterable[str]:
+    def tags_for_value(self, value: str) -> list[str]:
         """
         Return list of tags assigned to a value.
 

{dkist_processing_common-13.0.5rc2 → dkist_processing_common-13.0.6}/dkist_processing_common/models/graphql.py

@@ -53,13 +53,13 @@ class InputDatasetPartResponse(InputDatasetBaseModel):
     """Response class for the input dataset part entity."""
 
     inputDatasetPartId: int
-    # inputDatasetPartDocument : Json[InputDatasetPartDocumentList] # will work in gqlclient v2
+    # inputDatasetPartDocument : Json[InputDatasetPartDocumentList] # wishlist for next gqlclient version
     inputDatasetPartDocument: Json[list]
     inputDatasetPartType: InputDatasetPartTypeResponse
 
     @field_validator("inputDatasetPartDocument", mode="after")
     @classmethod
-    def _use_frame_or_parameter_model(cls, value_list):  # not needed for gqlclient v2
+    def _use_frame_or_parameter_model(cls, value_list):  # not needed if Json[model] implemented
         return InputDatasetPartDocumentList(doc_list=value_list)
 
 
@@ -121,12 +121,12 @@ class RecipeRunResponse(GraphqlBaseModel):
     recipeInstance: RecipeInstanceResponse
     recipeInstanceId: int
     recipeRunProvenances: list[RecipeRunProvenanceResponse]
-    # configuration: Json[RecipeRunConfiguration] | None # will work in gqlclient v2
+    # configuration: Json[RecipeRunConfiguration] | None # wishlist for next gqlclient version
     configuration: Json[dict] | None
 
     @field_validator("configuration", mode="after")
     @classmethod
-    def _use_recipe_run_configuration_model(cls, value):  # not needed for gqlclient v2
+    def _use_recipe_run_configuration_model(cls, value):
         if value is None:
             return RecipeRunConfiguration()
         return RecipeRunConfiguration.model_validate(value)

{dkist_processing_common-13.0.5rc2 → dkist_processing_common-13.0.6}/dkist_processing_common/models/input_dataset.py

@@ -56,7 +56,7 @@ class InputDatasetParameterValue(InputDatasetBaseModel):
     """Input dataset parameter value validator."""
 
     parameter_value_id: int
-    # parameter_value: Json[InputDatasetFilePointer] | Json[Any] # will work in gqlclient v2
+    # parameter_value: Json[InputDatasetFilePointer] | Json[Any] # wishlist for next gqlclient version
     parameter_value: Json[Any]
     parameter_value_start_date: Annotated[
         datetime, Field(default=datetime(1, 1, 1)), PlainSerializer(lambda x: x.isoformat())
@@ -64,7 +64,7 @@ class InputDatasetParameterValue(InputDatasetBaseModel):
 
     @field_validator("parameter_value", mode="after")
     @classmethod
-    def validate_parameter_value(cls, param_val):
+    def validate_parameter_value(cls, param_val):  # not needed if Json[model] implemented
         """Decode and provide additional validation for parameter_value types."""
         match param_val:
             case {"__file__": _}:

{dkist_processing_common-13.0.5rc2 → dkist_processing_common-13.0.6}/dkist_processing_common/tasks/base.py

@@ -214,10 +214,7 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
         """
         self.write_counter.add(amount=1, attributes=self.base_telemetry_attributes)
         file_obj = encoder(data, **encoder_kwargs)
-        if isinstance(tags, str):
-            tags = [tags]
-        else:
-            tags = [t for t in tags]  # copy the input list so we don't modify it in place
+        tags = self.scratch.parse_tags(tags)
         tags.append(Tag.workflow_task(self.__class__.__name__))
 
         relative_path = relative_path or self.build_generic_tag_filename(tags)
@@ -247,9 +244,9 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
             StemName.modstate.value,
         ]
 
-    def build_generic_tag_filename(self, tags: list) -> str:
+    def build_generic_tag_filename(self, tags: list[str]) -> str:
         """
-        Build a filename from a set of tags.
+        Build a filename from a list of tags.
 
         The algorithm is:
 
@@ -263,9 +260,8 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
 
          4. You can have any extension you want so long as it's ".dat".
         """
-        # This call copies the input list so it doesn't get modified in place and flattens the list to allow
-        # arbitrarily nested lists.
-        copied_tags = self.scratch.parse_tags(tags)
+        # Copy the tag list so it doesn't get modified in place.
+        copied_tags = tags.copy()
         try:
             copied_tags.remove(StemName.frame.value)
         except ValueError:

{dkist_processing_common-13.0.5rc2 → dkist_processing_common-13.0.6}/dkist_processing_common/tests/test_scratch.py

@@ -9,7 +9,7 @@ from uuid import uuid4
 import pytest
 
 from dkist_processing_common._util.scratch import WorkflowFileSystem
-from dkist_processing_common._util.scratch import _flatten_list
+from dkist_processing_common._util.scratch import _flatten_tags
 
 
 @pytest.fixture(
@@ -54,10 +54,10 @@ def test_workflow_file_system(workflow_file_system):
 @pytest.mark.parametrize(
     "tags",
     [
-        pytest.param(None, id="0"),
-        pytest.param("foo", id="1"),
-        pytest.param(["foo", "baz"], id="2"),
-        pytest.param(["foo", ["bar", "baz"]], id="nested"),
+        pytest.param([], id="empty_list_tag"),
+        pytest.param("foo", id="str_tag"),
+        pytest.param(["foo", "baz"], id="list_tag"),
+        pytest.param(["foo", ["bar", "baz"]], id="nested_list_tag"),
     ],
 )
 def test_workflow_file_system_write(workflow_file_system, tags, folder_path):
@@ -82,15 +82,28 @@ def test_workflow_file_system_write(workflow_file_system, tags, folder_path):
         assert next(wkflow_fs.find_all(tags=tags)) == full_file_path
 
 
-def test_workflow_file_system_write_invalid(workflow_file_system):
+def test_workflow_file_system_write_invalid_path(workflow_file_system):
     """
     Given: An instance of WorkflowFileSystem
     When: writing to an absolute path
-    Then: ValueError is raised
+    Then: the correct ValueError is raised
     """
     wkflow_fs, _, _ = workflow_file_system
-    with pytest.raises(ValueError):
-        wkflow_fs.write(b"1234", Path.cwd())
+    with pytest.raises(ValueError, match="relative"):
+        wkflow_fs.write(b"1234", Path.cwd(), tags="test_tag")
+
+
+def test_workflow_file_system_write_invalid_tags(workflow_file_system):
+    """
+    Given: An instance of WorkflowFileSystem
+    When: writing with tags=None
+    Then: the correct ValueError is raised
+    """
+    wkflow_fs, _, _ = workflow_file_system
+    file_path = Path(f"{uuid4().hex[:6]}.bin")
+    rel_path = Path("test_write_path") / file_path
+    with pytest.raises(TypeError, match="tags=None"):
+        wkflow_fs.write(b"1234", rel_path, tags=None)
 
 
 @pytest.mark.parametrize(
@@ -335,9 +348,9 @@ def test_workflow_file_system_duplicate_do_not_overwrite(workflow_file_system):
     file_path = Path(f"{uuid4().hex[:6]}.bin")
     rel_path = Path("test_write_path") / file_path
     # When / Then
-    wkflow_fs.write(file_obj, rel_path, tags=None, overwrite=False)
+    wkflow_fs.write(file_obj, rel_path, tags=[], overwrite=False)
     with pytest.raises(FileExistsError):
-        wkflow_fs.write(file_obj, rel_path, tags=None, overwrite=False)
+        wkflow_fs.write(file_obj, rel_path, tags=[], overwrite=False)
 
 
 def test_workflow_file_system_duplicate_do_overwrite(workflow_file_system):
@@ -372,17 +385,22 @@ def test_workflow_file_system_duplicate_do_overwrite(workflow_file_system):
             ["a", "b", "c", "d", "e", "f", "g", "h", "i"],
             id="Nested",
         ),
+        pytest.param(
+            {"a", ("b", ("c", "d"))},
+            ["a", "b", "c", "d"],
+            id="Nested_set",
+        ),
     ],
 )
-def test_flatten_list(given, expected):
+def test_flatten_tags(given, expected):
     """
-    Given: A list to flatten
-    When: Flattening the list
+    Given: An iterator to flatten
+    When: Flattening the iterator into a list
     Then: The new list matches the expected 1D result
     """
     # When
-    actual = _flatten_list(given)
-    assert actual == expected
+    actual = _flatten_tags(given)
+    assert sorted(actual) == sorted(expected)
 
 
 @pytest.fixture()

{dkist_processing_common-13.0.5rc2 → dkist_processing_common-13.0.6}/dkist_processing_common/tests/test_workflow_task_base.py

@@ -94,6 +94,20 @@ def test_valid_read_with_tag_object(workflow_data_task):
         assert tagged_filepath.exists()
 
 
+def test_valid_read_with_tag_set(workflow_data_task):
+    """
+    Given: a WorkflowDataTask with tagged data
+    When: reading tagged files using a set of tags
+    Then: the correct number of files are returned and they have the correct names
+    """
+    task, number_of_files, filenames, tag_string, tag_object = workflow_data_task
+    tagged_filepaths = list(task.read(tags={tag_string, tag_object}))
+    assert len(tagged_filepaths) == number_of_files
+    for tagged_filepath in tagged_filepaths:
+        assert tagged_filepath.name in filenames
+        assert tagged_filepath.exists()
+
+
 def test_valid_write(workflow_data_task):
     """
     Given: a WorkflowDataTask
@@ -111,14 +125,26 @@ def test_write_tags_is_none(workflow_data_task):
     """
     Given: a WorkflowDataTask
     When: writing a file to disk with tags=None
-    Then: a ValueError is raised
+    Then: the correct ValueError is raised
     """
     task, _, _, _, _ = workflow_data_task
     relative_path = "bytesio_path"
-    with pytest.raises(TypeError):
+    with pytest.raises(TypeError, match="tags=None"):
         task.write(data=bytes("abcdefg", "utf-8"), tags=None, relative_path=relative_path)
 
 
+def test_read_tags_is_none(workflow_data_task):
+    """
+    Given: a WorkflowDataTask
+    When: reading files with tags=None
+    Then: the correct ValueError is raised
+    """
+    task, _, _, _, _ = workflow_data_task
+    filepaths = task.read(tags=None)
+    with pytest.raises(TypeError, match="tags=None"):
+        next(filepaths)
+
+
 def test_read_nonexistent_tag(workflow_data_task):
     """
     Given: a WorkflowDataTask

{dkist_processing_common-13.0.5rc2 → dkist_processing_common-13.0.6}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 13.0.5rc2
+Version: 13.0.6
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -23,7 +23,7 @@ Requires-Dist: dkist-service-configuration<5.0,>=4.3.0
 Requires-Dist: dkist-spectral-lines<4.0,>=3.0.0
 Requires-Dist: solar-wavelength-calibration<3.0,>=2.0.0
 Requires-Dist: globus-sdk<5.0.0,>=4.0.0
-Requires-Dist: gqlclient[pydantic]==1.2.3
+Requires-Dist: gqlclient==2.0.0
 Requires-Dist: sqids==0.5.1
 Requires-Dist: matplotlib>=3.4
 Requires-Dist: moviepy>=2.0.0

{dkist_processing_common-13.0.5rc2 → dkist_processing_common-13.0.6}/dkist_processing_common.egg-info/SOURCES.txt

@@ -7,7 +7,6 @@ README.rst
 bitbucket-pipelines.yml
 pyproject.toml
 changelog/.gitempty
-changelog/329.feature.rst
 dkist_processing_common/__init__.py
 dkist_processing_common/config.py
 dkist_processing_common/manual.py

{dkist_processing_common-13.0.5rc2 → dkist_processing_common-13.0.6}/dkist_processing_common.egg-info/requires.txt

@@ -8,7 +8,7 @@ dkist-service-configuration<5.0,>=4.3.0
 dkist-spectral-lines<4.0,>=3.0.0
 solar-wavelength-calibration<3.0,>=2.0.0
 globus-sdk<5.0.0,>=4.0.0
-gqlclient[pydantic]==1.2.3
+gqlclient==2.0.0
 sqids==0.5.1
 matplotlib>=3.4
 moviepy>=2.0.0

{dkist_processing_common-13.0.5rc2 → dkist_processing_common-13.0.6}/pyproject.toml

@@ -31,7 +31,7 @@ dependencies = [
   "dkist-spectral-lines >= 3.0.0, <4.0",
   "solar-wavelength-calibration >= 2.0.0, <3.0",
   "globus-sdk >= 4.0.0, <5.0.0",
-  "gqlclient[pydantic] == 1.2.3",
+  "gqlclient == 2.0.0",
   "sqids == 0.5.1",
   "matplotlib >= 3.4",
   "moviepy >= 2.0.0",

dkist_processing_common-13.0.5rc2/changelog/329.feature.rst

@@ -1 +0,0 @@
-Remove 4 unused quality metric codes: RANGE, NOISE, SENSITIVITY, HISTORICAL.