dkist-processing-common 10.5.4__py3-none-any.whl → 12.1.0rc1__py3-none-any.whl

changelog/280.misc.rst

@@ -0,0 +1 @@
+Speed up the reading of INPUT files in Parse tasks by turning off image decompression and checksum checks.

changelog/282.feature.2.rst

@@ -0,0 +1,2 @@
+Add `SetStem` base class that has all the benefits of `ListStem` but also gets a speedup by storing values in a `set` for
+cases where repeated values don't need to be tracked.

changelog/282.feature.rst

@@ -0,0 +1,2 @@
+Add `ListStem` base class for huge speedup in cases where the keys don't matter and the `getter` logic only depends on the
+list of values computed by `setter`. This is the case for most (all?) "Buds".

changelog/284.feature.rst

@@ -0,0 +1 @@
+Speed up parsing of the `*CadenceBud`, `TaskDateBeginBud`, and `[Task]NearFloatBud` by basing these buds on `ListStem`.

changelog/285.feature.rst

@@ -0,0 +1,2 @@
+Speed up `NumCSStepBud`, `[Task]UniqueBud`, `[Task]ContributingIdsBud`, and `TaskRoundTimeBudBase` parsing by basing
+these buds on `SetStem`.

changelog/285.misc.rst

@@ -0,0 +1,2 @@
+Update `RetarderNameBud` to drop "clear" values (i.e., the retarder is out of the beam) in the `setter` instead of the `getter`.
+This brings it in line with standard Bud-practice.

changelog/286.feature.rst

@@ -0,0 +1,2 @@
+Speed up `CSStepFlower` parsing by using an internal set to keep track of the unique `CSStep` objects. This removes the
+need to compute the unique set when computing the tag for each file.

changelog/287.misc.rst

@@ -0,0 +1 @@
+Convert the TimeLookupBud to be a SetStem constant.

dkist_processing_common/__init__.py

@@ -1,4 +1,5 @@
 """Package providing support classes and methods used by all workflow tasks."""
+
 from importlib.metadata import PackageNotFoundError
 from importlib.metadata import version
 

dkist_processing_common/_util/constants.py

@@ -1,4 +1,5 @@
 """Wrapper for interactions with shared database that holds arbitrary data that persists across the entire recipe run."""
+
 import json
 from collections.abc import MutableMapping
 from enum import Enum

dkist_processing_common/_util/graphql.py

@@ -1,4 +1,5 @@
 """Extension of the GraphQL supporting retries for data processing use cases."""
+
 import logging
 from typing import Any
 from typing import Callable

dkist_processing_common/_util/scratch.py

@@ -1,4 +1,5 @@
 """Scratch file system api."""
+
 import logging
 from contextlib import contextmanager
 from os import umask
@@ -9,7 +10,6 @@ from typing import Generator
 from dkist_processing_common._util.tags import TagDB
 from dkist_processing_common.config import common_configurations
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -82,7 +82,7 @@ class WorkflowFileSystem:
         return self.workflow_base_path / relative_path
 
     @staticmethod
-    def _parse_tags(tags: str | list | None) -> list:
+    def parse_tags(tags: str | list | None) -> list:
         """Parse tags to support an individual tag in the form of a string or an arbitrarily nested list of strings."""
         if tags is None:
             return []
@@ -115,7 +115,7 @@ class WorkflowFileSystem:
         -------
         None
         """
-        tags = self._parse_tags(tags)
+        tags = self.parse_tags(tags)
         path = self.absolute_path(relative_path)
         # audit the path that was written to scratch
         self._audit_db.add(tag=self._audit_write_tag, value=str(path))
@@ -163,7 +163,7 @@ class WorkflowFileSystem:
         -------
         None
         """
-        tags = self._parse_tags(tags)
+        tags = self.parse_tags(tags)
         path = Path(path)
         if not (self.workflow_base_path in path.parents):
             raise ValueError(
@@ -201,7 +201,7 @@ class WorkflowFileSystem:
 
     def remove_tags(self, path: Path | str, tags: list | str) -> None:
         """Remove a tag or tags from a given path."""
-        tags = self._parse_tags(tags)
+        tags = self.parse_tags(tags)
         for tag in tags:
             self._tag_db.remove(tag, str(path))
 
@@ -218,7 +218,7 @@ class WorkflowFileSystem:
         -------
         A generator of path objects matching the union of the desired tags
         """
-        tags = self._parse_tags(tags)
+        tags = self.parse_tags(tags)
         paths = self._tag_db.any(tags)
         logger.debug(f"Found {len(paths)} files containing the set of {tags=}")
         for path in paths:
@@ -237,7 +237,7 @@ class WorkflowFileSystem:
         -------
         A generator of path objects matching the intersection of the desired tags
         """
-        tags = self._parse_tags(tags)
+        tags = self.parse_tags(tags)
         paths = self._tag_db.all(tags)
         logger.debug(f"Found {len(paths)} files containing the set of {tags=}")
         for path in paths:
@@ -256,7 +256,7 @@ class WorkflowFileSystem:
         -------
         The number of objects tagged with the union of the input tags.
         """
-        tags = self._parse_tags(tags)
+        tags = self.parse_tags(tags)
         return len(self._tag_db.any(tags))
 
     def count_all(self, tags: str | list) -> int:
@@ -273,7 +273,7 @@ class WorkflowFileSystem:
         The number of objects tagged with the intersection of the input tags.
 
         """
-        tags = self._parse_tags(tags)
+        tags = self.parse_tags(tags)
         return len(self._tag_db.all(tags))
 
     def close(self):

dkist_processing_common/_util/tags.py

@@ -1,4 +1,5 @@
 """Tag cloud manager."""
+
 from pathlib import Path
 from typing import Iterable
 

dkist_processing_common/codecs/array.py

@@ -0,0 +1,20 @@
+"""Encoder/decoder for writing/reading numpy arrays."""
+
+import io
+from pathlib import Path
+
+import numpy as np
+
+from dkist_processing_common.codecs.iobase import iobase_encoder
+
+
+def array_encoder(data: np.ndarray, **np_kwargs) -> bytes:
+    """Convert a numpy array to bytes compatible with np.load()."""
+    buffer = io.BytesIO()
+    np.save(buffer, data, **np_kwargs)
+    return iobase_encoder(buffer)
+
+
+def array_decoder(path: Path, **np_kwargs) -> np.ndarray:
+    """Return the data in the file as a numpy array using np.load()."""
+    return np.load(path, **np_kwargs)

dkist_processing_common/codecs/asdf.py

@@ -1,4 +1,5 @@
 """Encoders and decoders for writing and reading ASDF files."""
+
 from io import BytesIO
 from pathlib import Path
 
@@ -7,14 +8,19 @@ import asdf
 from dkist_processing_common.codecs.iobase import iobase_encoder
 
 
-def asdf_encoder(tree: dict, custom_schema=None, **asdf_write_kwargs) -> bytes:
-    """Convert a dict to raw bytes representing an ASDF file for writing to a file."""
-    asdf_obj = asdf.AsdfFile(tree, custom_schema=custom_schema)
+def asdf_fileobj_encoder(asdf_obj: asdf.AsdfFile, custom_schema=None, **asdf_write_kwargs) -> bytes:
+    """Save an `asdf.AsdfFile` object."""
     file_obj = BytesIO()
     asdf_obj.write_to(file_obj, **asdf_write_kwargs)
     return iobase_encoder(file_obj)
 
 
+def asdf_encoder(tree: dict, custom_schema=None, **asdf_write_kwargs) -> bytes:
+    """Convert a dict to raw bytes representing an ASDF file for writing to a file."""
+    asdf_obj = asdf.AsdfFile(tree, custom_schema=custom_schema)
+    return asdf_fileobj_encoder(asdf_obj, custom_schema=custom_schema, **asdf_write_kwargs)
+
+
 def asdf_decoder(
     path: Path, lazy_load: bool = False, memmap: bool = False, **asdf_read_kwargs
 ) -> dict:

dkist_processing_common/codecs/basemodel.py

@@ -0,0 +1,22 @@
+"""Encoder/decoder for writing and reading Pydantic BaseModel objects."""
+
+from pathlib import Path
+from typing import Type
+
+from pydantic import BaseModel
+
+from dkist_processing_common.codecs.bytes import bytes_decoder
+from dkist_processing_common.codecs.str import str_encoder
+
+
+def basemodel_encoder(data: BaseModel, **basemodel_kwargs) -> bytes:
+    """Convert a Pydantic BaseModel object into bytes for writing to file."""
+    data_dump = data.model_dump_json(**basemodel_kwargs)
+    return str_encoder(data_dump)
+
+
+def basemodel_decoder(path: Path, model: Type[BaseModel], **basemodel_kwargs) -> BaseModel:
+    """Return the data in the file as a Pydantic BaseModel object."""
+    data = bytes_decoder(path)
+    model_validated = model.model_validate_json(data, **basemodel_kwargs)
+    return model_validated

dkist_processing_common/codecs/bytes.py

@@ -1,4 +1,5 @@
 """Encoder/decoder for writing and reading bytes objects."""
+
 from pathlib import Path
 
 

dkist_processing_common/codecs/fits.py

@@ -1,4 +1,5 @@
 """Encoders and decoders for writing and reading FITS files."""
+
 from io import BytesIO
 from pathlib import Path
 from typing import Type
@@ -30,15 +31,30 @@ def fits_hdulist_encoder(hdu_list: fits.HDUList) -> bytes:
     return iobase_encoder(file_obj)
 
 
-def fits_hdu_decoder(path: Path) -> fits.PrimaryHDU | fits.CompImageHDU:
+def fits_hdu_decoder(
+    path: Path,
+    hdu: int | None = None,
+    checksum: bool = True,
+    disable_image_compression: bool = False,
+) -> fits.PrimaryHDU | fits.CompImageHDU:
     """Read a Path with `fits` to produce an `HDUList`."""
-    hdu_list = fits.open(path)
-    return _extract_hdu(hdu_list)
+    hdu_list = fits.open(
+        path, checksum=checksum, disable_image_compression=disable_image_compression
+    )
+    return _extract_hdu(hdu_list, hdu)
 
 
-def fits_array_decoder(path: Path, auto_squeeze: bool = True) -> np.ndarray:
+def fits_array_decoder(
+    path: Path,
+    hdu: int | None = None,
+    auto_squeeze: bool = True,
+    checksum: bool = True,
+    disable_image_compression: bool = False,
+) -> np.ndarray:
     """Read a Path with `fits` and return the `.data` property."""
-    hdu = fits_hdu_decoder(path)
+    hdu = fits_hdu_decoder(
+        path, hdu=hdu, checksum=checksum, disable_image_compression=disable_image_compression
+    )
     data = hdu.data
 
     # This conditional is explicitly to catch summit data with a dummy first axis for WCS
@@ -49,15 +65,27 @@ def fits_array_decoder(path: Path, auto_squeeze: bool = True) -> np.ndarray:
 
 
 def fits_access_decoder(
-    path: Path, fits_access_class: Type[FitsAccessBase], **fits_access_kwargs
+    path: Path,
+    fits_access_class: Type[FitsAccessBase],
+    checksum: bool = True,
+    disable_image_compression: bool = False,
+    **fits_access_kwargs,
 ) -> FitsAccessBase:
     """Read a Path with `fits` and ingest into a `FitsAccessBase`-type object."""
-    hdu = fits_hdu_decoder(path)
+    hdu = fits_hdu_decoder(
+        path, checksum=checksum, disable_image_compression=disable_image_compression
+    )
     return fits_access_class(hdu=hdu, name=str(path), **fits_access_kwargs)
 
 
-def _extract_hdu(hdul: fits.HDUList) -> fits.PrimaryHDU | fits.CompImageHDU:
-    """Return the fits hdu associated with the data in the hdu list."""
+def _extract_hdu(hdul: fits.HDUList, hdu: int | None = None) -> fits.PrimaryHDU | fits.CompImageHDU:
+    """
+    Return the fits hdu associated with the data in the hdu list.
+
+    Only search down the hdu index for the data if the hdu index is not explicitly provided.
+    """
+    if hdu is not None:
+        return hdul[hdu]
     if hdul[0].data is not None:
         return hdul[0]
     return hdul[1]

dkist_processing_common/codecs/iobase.py

@@ -1,4 +1,5 @@
 """Encoder/decoder for writing and reading IOBase binary (i.e., not Text) objects."""
+
 from io import BytesIO
 from io import IOBase
 from io import TextIOBase

dkist_processing_common/codecs/json.py

@@ -1,4 +1,5 @@
 """Encoder/decoders for writing and reading JSON files."""
+
 import json
 from pathlib import Path
 from typing import Any

dkist_processing_common/codecs/path.py

@@ -1,4 +1,5 @@
 """Default decoder to pass through paths from `read`."""
+
 from pathlib import Path
 
 

dkist_processing_common/codecs/quality.py

@@ -1,4 +1,5 @@
 """Encoder/decoders for writing and reading quality data."""
+
 import json
 import logging
 from datetime import datetime
@@ -10,7 +11,6 @@ import numpy as np
 from dkist_processing_common.codecs.json import json_decoder
 from dkist_processing_common.codecs.json import json_encoder
 
-
 logger = logging.getLogger(__name__)
 
 

dkist_processing_common/codecs/str.py

@@ -1,4 +1,5 @@
 """Encoder/decoder for writing and reading str to files."""
+
 from pathlib import Path
 
 from dkist_processing_common.codecs.bytes import bytes_decoder

dkist_processing_common/config.py

@@ -1,4 +1,5 @@
 """Common configurations."""
+
 from dkist_processing_core.config import DKISTProcessingCoreConfiguration
 from dkist_service_configuration.settings import DEFAULT_MESH_SERVICE
 from dkist_service_configuration.settings import MeshService
@@ -9,41 +10,80 @@ from talus import ConsumerConnectionParameterFactory
 from talus import ProducerConnectionParameterFactory
 
 
-class RetryConfig(BaseModel):
-    """Retry metadata model."""
+class GlobusClientCredential(BaseModel):
+    """Globus client credential."""
 
-    retry_delay: int = 1
-    retry_backoff: int = 2
-    retry_jitter: tuple[int, int] = (1, 10)
-    retry_max_delay: int = 300
-    retry_tries: int = -1
+    client_id: str = Field(..., description="Globus client ID for transfers.")
+    client_secret: str = Field(..., description="Globus client secret for transfers.")
 
 
 class DKISTProcessingCommonConfiguration(DKISTProcessingCoreConfiguration):
     """Common configurations."""
 
-    retry_config: RetryConfig = Field(default_factory=RetryConfig)
     # metadata-store-api
-    gql_auth_token: str | None = None
+    gql_auth_token: str | None = Field(
+        default="dev", description="The auth token for the metadata-store-api."
+    )
     # object-store-api
-    object_store_access_key: str | None = None
-    object_store_secret_key: str | None = None
-    object_store_use_ssl: bool = False
-    multipart_threshold: int | None = None
-    s3_client_config: dict | None = None
-    s3_upload_config: dict | None = None
-    s3_download_config: dict | None = None
+    object_store_access_key: str | None = Field(
+        default=None, description="The access key for the object store."
+    )
+    object_store_secret_key: str | None = Field(
+        default=None, description="The secret key for the object store."
+    )
+    object_store_use_ssl: bool = Field(
+        default=False, description="Whether to use SSL for the object store connection."
+    )
+    # start object-clerk library
+    multipart_threshold: int | None = Field(
+        default=None, description="Multipart threshold for the object store."
+    )
+    s3_client_config: dict | None = Field(
+        default=None, description="S3 client configuration for the object store."
+    )
+    s3_upload_config: dict | None = Field(
+        default=None, description="S3 upload configuration for the object store."
+    )
+    s3_download_config: dict | None = Field(
+        default=None, description="S3 download configuration for the object store."
+    )
     # globus
-    globus_transport_params: dict = Field(default_factory=dict)
-    globus_client_id: str | None = None
-    globus_client_secret: str | None = None
-    object_store_endpoint: str | None = None
-    scratch_endpoint: str | None = None
+    globus_max_retries: int = Field(
+        default=5, description="Max retries for transient errors on calls to the globus api."
+    )
+    globus_inbound_client_credentials: list[GlobusClientCredential] = Field(
+        default_factory=list,
+        description="Globus client credentials for inbound transfers.",
+        examples=[
+            [
+                {"client_id": "id1", "client_secret": "secret1"},
+                {"client_id": "id2", "client_secret": "secret2"},
+            ],
+        ],
+    )
+    globus_outbound_client_credentials: list[GlobusClientCredential] = Field(
+        default_factory=list,
+        description="Globus client credentials for outbound transfers.",
+        examples=[
+            [
+                {"client_id": "id3", "client_secret": "secret3"},
+                {"client_id": "id4", "client_secret": "secret4"},
+            ],
+        ],
+    )
+    object_store_endpoint: str | None = Field(
+        default=None, description="Object store Globus Endpoint ID."
+    )
+    scratch_endpoint: str | None = Field(default=None, description="Scratch Globus Endpoint ID.")
     # scratch
-    scratch_base_path: str = Field(default="scratch/")
-    scratch_inventory_db_count: int = 16
+    scratch_base_path: str = Field(default="scratch/", description="Base path for scratch storage.")
+    scratch_inventory_db_count: int = Field(
+        default=16, description="Number of databases in the scratch inventory (redis)."
+    )
     # docs
-    docs_base_url: str = Field(default="my_test_url")
+    docs_base_url: str = Field(
+        default="my_test_url", description="Base URL for the documentation site."
+    )
 
     @property
     def metadata_store_api_base(self) -> str:
@@ -105,4 +145,3 @@ class DKISTProcessingCommonConfiguration(DKISTProcessingCoreConfiguration):
 
 
 common_configurations = DKISTProcessingCommonConfiguration()
-common_configurations.log_configurations()

dkist_processing_common/manual.py

@@ -1,21 +1,19 @@
 """Task wrapper for manual execution outside the workflow engine."""
+
 import json
 import logging
 import shutil
-from dataclasses import asdict
-from io import BytesIO
 from pathlib import Path
 from typing import Callable
 from unittest.mock import patch
 
 from dkist_processing_core.task import TaskBase
 
-from dkist_processing_common.codecs.json import json_encoder
+from dkist_processing_common.codecs.basemodel import basemodel_encoder
 from dkist_processing_common.models.graphql import RecipeRunProvenanceMutation
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.base import WorkflowTaskBase
-from dkist_processing_common.tests.conftest import FakeGQLClient
-
+from dkist_processing_common.tests.mock_metadata_store import fake_gql_client_factory
 
 logger = logging.getLogger(__name__)
 
@@ -69,7 +67,7 @@ class ManualProcessing:
 
             with patch(
                 "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
-                new=FakeGQLClient,
+                new=fake_gql_client_factory(),
             ) as foo:
                 # Run the task with a FakeGQLClient. This will handle pre_run(), run(), and post_run()
                 with patch(
@@ -182,8 +180,8 @@ def writing_metadata_store_record_provenance(self, is_task_manual: bool, library
         workflowVersion=self.workflow_version,
     )
     self.write(
-        data=asdict(params),
-        encoder=json_encoder,
+        data=params,
+        encoder=basemodel_encoder,
         tags=["PROVENANCE_RECORD"],
         relative_path=f"{self.task_name}_provenance.json",
         overwrite=True,