dkist-header-validator 4.1.0__tar.gz → 5.0.0__tar.gz

{dkist-header-validator-4.1.0/dkist_header_validator.egg-info → dkist-header-validator-5.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-header-validator
-Version: 4.1.0
+Version: 5.0.0
 Summary: DKIST data validator
 Home-page: https://bitbucket.org/dkistdc/dkist-header-validator/src/main/
 Author: NSO / AURA

{dkist-header-validator-4.1.0 → dkist-header-validator-5.0.0}/bitbucket-pipelines.yml

@@ -49,13 +49,14 @@ definitions:
 pipelines:
   default:
     - step: *lint
-    - step: *test
-    - step: *docs
-    - step: *scan
+    - parallel:
+        - step: *test
+        - step: *docs
+        - step: *scan
   tags:
     'v*':
+      - step: *lint
       - parallel:
-          - step: *lint
           - step: *test
           - step: *docs
           - step: *scan

{dkist-header-validator-4.1.0 → dkist-header-validator-5.0.0}/dkist_header_validator/base_validator.py

@@ -9,7 +9,9 @@ from numbers import Integral
 from numbers import Real
 from pathlib import Path
 from typing import Any
+from typing import Callable
 from typing import IO
+from typing import Optional
 from typing import Type
 
 import astropy.time as t
@@ -453,21 +455,14 @@ class SpecValidator:
         """
         try:
             with fits.open(input_headers) as hdul:
-                if len(hdul) > 2:
-                    raise ValidationException(
-                        "Too many HDUs in your HDUList! May only have two HDUs at most."
-                    )
                 # verify fits headers with astropy verify library
                 hdul.verify("exception")
                 # normalize headers into a dict
                 try:
-                    hdus = self._headers_to_dict(hdul[1].header)
                     data = hdul[1].data
                 except IndexError:  # non-compressed
-                    hdus = self._headers_to_dict(hdul[0].header)
                     data = hdul[0].data
-                fits_cards = self._capture_fits_cards(hdus)
-                verified_headers = self.verify_headers(hdus, extra)
+                verified_headers, fits_cards = self._validate_headers(hdul, extra)
                 return verified_headers, fits_cards, data
         except (ValueError, FileNotFoundError, OSError, IndexError) as exc:
             logger.debug(f"Cannot parse headers: detail = {exc}")
@@ -702,3 +697,56 @@ class SpecValidator:
             return self._format_output(
                 return_type, translated_headers, input_headers, data, fits_cards
             )
+
+
+class ProcessedSpecValidator(SpecValidator):
+    """
+    Validates FITS Headers against a schema that is updated based on the actual headers.
+
+    The two current examples of a "processed" spec are keys that are updated based on expansion or on conditional
+    requiredness.
+
+    Parameters
+    ----------
+    spec_processor_function
+        A function that can process a spec based on an input header. Probably `load_processed_spec214`.
+
+    SchemaValidationException
+        SpecValidationException or subclass of SpecValidationException
+            to raise if spec_validator validation fails
+
+    """
+
+    def __init__(
+        self,
+        spec_processor_function: Callable,
+        SchemaValidationException: Type[SpecValidationException] = SpecValidationException,
+    ):
+        self.spec_processor_function = spec_processor_function
+
+        # Use an unprocessed (i.e., no header given) spec to initialize the base Validator.
+        base_schema = self.spec_processor_function()
+        super().__init__(
+            spec_schema=base_schema, SchemaValidationException=SchemaValidationException
+        )
+
+    def verify_headers(self, headers, extra) -> dict:
+        """
+        Validates file headers against the instance spec_validator
+
+        Parameters
+        ----------
+        headers
+            file headers
+        extra
+            switch for validation to allow extra keys in schema
+        Returns
+        -------
+        dict of headers
+
+        Raises
+        ------
+        SchemaValidationException
+        """
+        self.spec_schema = SpecSchema(self.spec_processor_function(**headers))
+        return super().verify_headers(headers, extra)

{dkist-header-validator-4.1.0 → dkist-header-validator-5.0.0}/dkist_header_validator/spec_validators.py

@@ -4,7 +4,9 @@ Validators configured for specific Fits Specs
 from dkist_fits_specifications import spec122
 from dkist_fits_specifications import spec214
 from dkist_fits_specifications.spec214 import level0
+from dkist_fits_specifications.spec214 import load_processed_spec214
 
+from dkist_header_validator.base_validator import ProcessedSpecValidator
 from dkist_header_validator.base_validator import SpecValidator
 from dkist_header_validator.exceptions import SpecSchemaDefinitionException
 from dkist_header_validator.exceptions import SpecValidationException
@@ -49,7 +51,7 @@ spec214_l0_validator = SpecValidator(
     SchemaValidationException=Spec214ValidationException,
 )
 
-spec214_validator = SpecValidator(
-    spec_schema=spec214.load_spec214(),
+spec214_validator = ProcessedSpecValidator(
+    spec_processor_function=load_processed_spec214,
     SchemaValidationException=Spec214ValidationException,
 )

{dkist-header-validator-4.1.0 → dkist-header-validator-5.0.0}/dkist_header_validator/tests/conftest.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from random import choice
 from uuid import uuid4
 
 import numpy as np
@@ -7,6 +8,7 @@ from astropy.io import fits
 from astropy.wcs import WCS
 from dkist_data_simulator.spec122 import Spec122Dataset
 from dkist_data_simulator.spec214 import Spec214Dataset
+from dkist_fits_specifications.spec214 import load_spec214
 
 FITS_OBJECT_TYPES = [
     "fits",
@@ -536,7 +538,6 @@ class BaseSpec214DatasetCaseSensitive(Spec214Dataset):
             time_delta=1,
             instrument=instrument,
         )
-        self.add_remove_key("VBISYNCM")
         self.add_constant_key("VBISYNCM", "fIxED")
 
     @property
@@ -556,3 +557,38 @@ def valid_spec_214_casesensitive(tmpdir, request):
     yield get_fits_object(
         object_type=request.param, tmpdir=tmpdir, ds=BaseSpec214DatasetCaseSensitive()
     )
+
+
+class InvalidPolarimetricSpec214Dataset(BaseSpec214Dataset):
+    def __init__(self):
+        super().__init__(instrument="visp")
+        self.add_constant_key("VSPPOLMD", "observe_polarimetric")
+        self.add_remove_key("POL_SENS")
+
+
+@pytest.fixture(scope="function", params=FITS_OBJECT_TYPES)
+def invalid_polarimetric_spec_214_object(tmpdir, request):
+    yield get_fits_object(
+        object_type=request.param, tmpdir=tmpdir, ds=InvalidPolarimetricSpec214Dataset()
+    )
+
+
+class InvalidInstrumentTableSpec214Dataset(BaseSpec214Dataset):
+    def __init__(self, instrument: str):
+        super().__init__(instrument=instrument)
+
+        # Remove a random key from the instrument table
+        glob_name = instrument.lower().replace(
+            "-", ""
+        )  # For cryo and dl b/c the yamls don't have dashes
+        instrument_keys = load_spec214(glob=glob_name)[glob_name]
+        removal_candidates = [k for k, v in instrument_keys.items() if "instrument_required" in v]
+        removed_key = choice(removal_candidates)
+        self.add_remove_key(removed_key)
+        self.removed_key = removed_key
+
+
+@pytest.fixture(scope="function", params=FITS_OBJECT_TYPES)
+def invalid_instrument_table_spec_214_object(tmpdir, request, instrument):
+    dataset = InvalidInstrumentTableSpec214Dataset(instrument)
+    yield get_fits_object(object_type=request.param, tmpdir=tmpdir, ds=dataset), dataset.removed_key

{dkist-header-validator-4.1.0 → dkist-header-validator-5.0.0}/dkist_header_validator/tests/test_spec214_validation-.py

@@ -3,10 +3,12 @@ from pathlib import Path
 
 import numpy as np
 import pytest
+from dkist_fits_specifications.spec214 import load_processed_spec214
 
 from dkist_header_validator import spec214_l0_validator
 from dkist_header_validator import spec214_validator
 from dkist_header_validator import Spec214ValidationException
+from dkist_header_validator.base_validator import ProcessedSpecValidator
 from dkist_header_validator.exceptions import ReturnTypeException
 from dkist_header_validator.exceptions import ValidationException
 
@@ -168,3 +170,30 @@ def test_validate_toomanyHDUs(valid_spec_214_too_many_HDUs):
     # raises exception on failure
     with pytest.raises(ValidationException):
         spec214_validator.validate(valid_spec_214_too_many_HDUs)
+
+
+def test_polarimetric_required_key_missing(invalid_polarimetric_spec_214_object):
+    """
+    Given: Polarimetric headers with a missing `polarimetric_required` key
+    When: Validating headers
+    Then: The correct Error is raised
+    """
+    with pytest.raises(
+        Spec214ValidationException,
+        match="'POL_SENS': 'required key not provided. Required keyword not present'",
+    ):
+        spec214_validator.validate(invalid_polarimetric_spec_214_object)
+
+
+@pytest.mark.parametrize("instrument", ["cryo-nirsp", "dlnirsp", "vbi", "visp", "vtf"])
+def test_instrument_required_key_missing(invalid_instrument_table_spec_214_object):
+    """
+    Given: Headers from a specific instrument, but with one of the required header keys removed
+    When: Validating headers
+    Then: The correct Error is raised
+    """
+    fits_object, missing_key = invalid_instrument_table_spec_214_object
+    with pytest.raises(
+        Spec214ValidationException, match=f"'{missing_key}': 'required key not provided"
+    ):
+        spec214_validator.validate(fits_object)

{dkist-header-validator-4.1.0 → dkist-header-validator-5.0.0}/dkist_header_validator/translator.py

@@ -7,7 +7,7 @@ from typing import IO
 from astropy.io import fits
 from astropy.io.fits.hdu.hdulist import HDUList
 from dkist_fits_specifications.spec214 import level0
-from dkist_fits_specifications.spec214 import load_expanded_spec214
+from dkist_fits_specifications.spec214 import load_processed_spec214
 from dkist_fits_specifications.spec214 import load_spec214
 
 from dkist_header_validator.utils.expansions import expand_index_d
@@ -67,7 +67,7 @@ def sanitize_to_spec214_level1(
     input_headers, input_data = _parse_fits_like_input(input_headers)
     header = fits.Header(input_headers)
     # convert headers
-    expanded_214 = load_expanded_spec214(**dict(input_headers))
+    expanded_214 = load_processed_spec214(**dict(input_headers))
     all_214_keys = reduce(list.__add__, map(list, expanded_214.values()))
 
     for keyword in tuple(header.keys()):

{dkist-header-validator-4.1.0 → dkist-header-validator-5.0.0}/dkist_header_validator/utils/expansions.py

@@ -1,6 +1,6 @@
-from dkist_fits_specifications.utils import expand_schema
-from dkist_fits_specifications.utils import ExpansionIndex
 from dkist_fits_specifications.utils import schema_type_hint
+from dkist_fits_specifications.utils.spec_processors.expansion import expand_schema
+from dkist_fits_specifications.utils.spec_processors.expansion import ExpansionIndex
 
 
 def expand_naxis(naxis: int, schema: schema_type_hint) -> schema_type_hint:

{dkist-header-validator-4.1.0 → dkist-header-validator-5.0.0}/dkist_header_validator/version.py

@@ -5,4 +5,4 @@ try:
     from setuptools_scm import get_version
     __version__ = get_version(root='..', relative_to=__file__)
 except Exception:
-    __version__ = '4.1.0'
+    __version__ = '5.0.0'

{dkist-header-validator-4.1.0 → dkist-header-validator-5.0.0/dkist_header_validator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist-header-validator
-Version: 4.1.0
+Version: 5.0.0
 Summary: DKIST data validator
 Home-page: https://bitbucket.org/dkistdc/dkist-header-validator/src/main/
 Author: NSO / AURA

{dkist-header-validator-4.1.0 → dkist-header-validator-5.0.0}/dkist_header_validator.egg-info/requires.txt

@@ -1,7 +1,7 @@
 astropy>=5.0
 voluptuous<1.0.0,>=0.11.7
 pyyaml>=6.0
-dkist-fits-specifications>=2.0.0
+dkist-fits-specifications>=4.0.0
 
 [cli]
 typer

{dkist-header-validator-4.1.0 → dkist-header-validator-5.0.0}/setup.cfg

@@ -22,7 +22,7 @@ install_requires =
 	astropy >= 5.0
 	voluptuous >=0.11.7, < 1.0.0
 	pyyaml >=6.0
-	dkist-fits-specifications >= 2.0.0
+	dkist-fits-specifications >= 4.0.0
 
 [options.extras_require]
 test =