dkist-processing-common 14.9.0__tar.gz → 14.9.1rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/PKG-INFO +1 -1
- dkist_processing_common-14.9.1rc1/changelog/358.misc.rst +1 -0
- dkist_processing_common-14.9.1rc1/changelog/363.feature.rst +1 -0
- dkist_processing_common-14.9.1rc1/changelog/363.misc.rst +1 -0
- dkist_processing_common-14.9.1rc1/dkist_processing_common/codecs/basemodel.py +41 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/quality.py +12 -1
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/l1_output_data.py +8 -13
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/mixin/quality.py +2 -7
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/quality_metrics.py +4 -8
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/trial_catalog.py +25 -6
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_assemble_quality.py +2 -5
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_codecs.py +180 -10
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_quality.py +19 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_quality_mixin.py +0 -11
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_trial_catalog.py +1 -4
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common.egg-info/PKG-INFO +1 -1
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common.egg-info/SOURCES.txt +3 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common.egg-info/scm_file_list.json +3 -0
- dkist_processing_common-14.9.1rc1/dkist_processing_common.egg-info/scm_version.json +8 -0
- dkist_processing_common-14.9.0/dkist_processing_common/codecs/basemodel.py +0 -22
- dkist_processing_common-14.9.0/dkist_processing_common.egg-info/scm_version.json +0 -8
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/.gitignore +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/.pre-commit-config.yaml +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/.readthedocs.yml +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/.rovodev/.review-agent.md +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/.snyk +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/CHANGELOG.rst +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/README.rst +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/bitbucket-pipelines.yml +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/changelog/.gitempty +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/__init__.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/_util/__init__.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/_util/constants.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/_util/graphql.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/_util/scratch.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/_util/tags.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/codecs/__init__.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/codecs/array.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/codecs/asdf.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/codecs/bytes.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/codecs/fits.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/codecs/iobase.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/codecs/json.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/codecs/path.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/codecs/quality.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/codecs/str.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/config.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/fonts/Lato-Regular.ttf +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/fonts/__init__.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/manual.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/__init__.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/constants.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/dkist_location.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/extras.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/fits_access.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/flower_pot.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/fried_parameter.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/graphql.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/input_dataset.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/message.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/message_queue_binding.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/metric_code.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/parameters.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/tags.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/task_name.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/telemetry.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/models/wavelength.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/__init__.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/average_bud.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/cs_step.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/dsps_repeat.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/experiment_id_bud.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/id_bud.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/l0_fits_access.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/l1_fits_access.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/lookup_bud.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/near_bud.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/observing_program_id_bud.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/proposal_id_bud.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/quality.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/retarder.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/single_value_single_key_flower.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/task.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/time.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/unique_bud.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/parsers/wavelength.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/__init__.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/assemble_movie.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/base.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/delete_input_frames.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/mixin/__init__.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/mixin/globus.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/mixin/interservice_bus.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/mixin/metadata_store.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/mixin/object_store.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/output_data_base.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/parse_l0_input_data.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/teardown.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/transfer_input_data.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/trial_output_data.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/write_extra.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/write_l1.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tasks/write_l1_base.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/__init__.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/conftest.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/mock_metadata_store.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_assemble_movie.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_base.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_constants.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_construct_dataset_extras.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_cs_step.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_delete_input_frames.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_dkist_location.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_fits_access.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_flower_pot.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_fried_parameter.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_input_dataset.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_interservice_bus.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_interservice_bus_mixin.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_manual_processing.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_output_data_base.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_parameters.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_parse_l0_input_data.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_publish_catalog_messages.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_scratch.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_stems.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_submit_dataset_metadata.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_tags.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_task_name.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_task_parsing.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_teardown.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_transfer_input_data.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_transfer_l1_output_data.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_trial_output_data.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_workflow_task_base.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common/tests/test_write_l1.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common.egg-info/dependency_links.txt +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common.egg-info/requires.txt +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/dkist_processing_common.egg-info/top_level.txt +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/docs/Makefile +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/docs/changelog.rst +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/docs/conf.py +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/docs/index.rst +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/docs/landing_page.rst +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/docs/make.bat +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/docs/requirements.txt +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/licenses/LICENSE.rst +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/pyproject.toml +0 -0
- {dkist_processing_common-14.9.0 → dkist_processing_common-14.9.1rc1}/setup.cfg +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Migrate `_format_warnings` from the quality mixin to the `QualityMetric` `BaseModel`.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Add codec support for `list[BaseModel]`.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Assemble quality data as list of `QualityMetric`.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Encoder/decoder for writing and reading Pydantic BaseModel objects."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Type
|
|
5
|
+
|
|
6
|
+
from pydantic import BaseModel
|
|
7
|
+
from pydantic import SerializeAsAny
|
|
8
|
+
from pydantic import TypeAdapter
|
|
9
|
+
|
|
10
|
+
from dkist_processing_common.codecs.bytes import bytes_decoder
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def basemodel_encoder(data: BaseModel | list[BaseModel | None], **basemodel_kwargs) -> bytes:
|
|
14
|
+
"""Convert a Pydantic BaseModel object into bytes for writing to file."""
|
|
15
|
+
# 1) TypeAdapter to handle either BaseModel or list[BaseModel]
|
|
16
|
+
# 2) SerializeAsAny to properly serialize any subclass of BaseModel IAW with its definition
|
|
17
|
+
adapter = TypeAdapter(SerializeAsAny[BaseModel] | list[SerializeAsAny[BaseModel]])
|
|
18
|
+
# directly from BaseModel to bytes
|
|
19
|
+
# uses utf-8 encoding
|
|
20
|
+
# equivalent to errors="strict"
|
|
21
|
+
return adapter.dump_json(data, **basemodel_kwargs)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def basemodel_decoder(path: Path, model: Type[BaseModel], **basemodel_kwargs) -> BaseModel:
|
|
25
|
+
"""Return the data in the file as a Pydantic BaseModel object."""
|
|
26
|
+
data = bytes_decoder(path)
|
|
27
|
+
# TypeAdapter to handle BaseModel
|
|
28
|
+
adapter = TypeAdapter(model)
|
|
29
|
+
model_validated = adapter.validate_json(data, **basemodel_kwargs)
|
|
30
|
+
return model_validated
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def list_of_basemodel_decoder(
|
|
34
|
+
path: Path, model: Type[BaseModel], **basemodel_kwargs
|
|
35
|
+
) -> list[BaseModel | None]:
|
|
36
|
+
"""Return the data in the file as a list of Pydantic BaseModel objects."""
|
|
37
|
+
data = bytes_decoder(path)
|
|
38
|
+
# TypeAdapter to handle list[BaseModel]
|
|
39
|
+
adapter = TypeAdapter(list[model | None])
|
|
40
|
+
model_validated = adapter.validate_json(data, **basemodel_kwargs)
|
|
41
|
+
return model_validated
|
|
@@ -347,6 +347,17 @@ QualityFacet = Annotated[
|
|
|
347
347
|
]
|
|
348
348
|
|
|
349
349
|
|
|
350
|
+
def format_warnings(v: Any) -> Any:
|
|
351
|
+
"""Empty list is None."""
|
|
352
|
+
return None if v == [] else v
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
QualityWarnings = Annotated[
|
|
356
|
+
list[str] | None,
|
|
357
|
+
BeforeValidator(format_warnings),
|
|
358
|
+
]
|
|
359
|
+
|
|
360
|
+
|
|
350
361
|
class QualityMetric(QualityModel):
|
|
351
362
|
"""Quality Metric."""
|
|
352
363
|
|
|
@@ -364,7 +375,7 @@ class QualityMetric(QualityModel):
|
|
|
364
375
|
modmat_data: ModMatData | None = None
|
|
365
376
|
raincloud_data: RaincloudData | None = None
|
|
366
377
|
efficiency_data: EfficiencyData | None = None
|
|
367
|
-
warnings:
|
|
378
|
+
warnings: QualityWarnings | None = None
|
|
368
379
|
|
|
369
380
|
|
|
370
381
|
###########################################################
|
|
@@ -7,7 +7,6 @@ from typing import Iterable
|
|
|
7
7
|
|
|
8
8
|
from dkist_processing_common.codecs.basemodel import basemodel_decoder
|
|
9
9
|
from dkist_processing_common.codecs.basemodel import basemodel_encoder
|
|
10
|
-
from dkist_processing_common.codecs.quality import quality_data_encoder
|
|
11
10
|
from dkist_processing_common.models.message import CatalogFrameMessage
|
|
12
11
|
from dkist_processing_common.models.message import CatalogFrameMessageBody
|
|
13
12
|
from dkist_processing_common.models.message import CatalogObjectMessage
|
|
@@ -19,7 +18,6 @@ from dkist_processing_common.models.quality import TaskCount
|
|
|
19
18
|
from dkist_processing_common.models.tags import Tag
|
|
20
19
|
from dkist_processing_common.tasks.mixin.globus import GlobusMixin
|
|
21
20
|
from dkist_processing_common.tasks.mixin.interservice_bus import InterserviceBusMixin
|
|
22
|
-
from dkist_processing_common.tasks.mixin.quality import QualityMixin
|
|
23
21
|
from dkist_processing_common.tasks.output_data_base import OutputDataBase
|
|
24
22
|
from dkist_processing_common.tasks.output_data_base import TransferDataBase
|
|
25
23
|
|
|
@@ -118,7 +116,7 @@ class TransferL1Data(TransferDataBase, GlobusMixin):
|
|
|
118
116
|
)
|
|
119
117
|
return
|
|
120
118
|
|
|
121
|
-
if count := len(paths) > 1:
|
|
119
|
+
if (count := len(paths)) > 1:
|
|
122
120
|
# dataset inventory does not support multiple quality data object keys
|
|
123
121
|
raise RuntimeError(
|
|
124
122
|
f"Found multiple quality data files to upload. Not supported."
|
|
@@ -156,13 +154,15 @@ class TransferL1Data(TransferDataBase, GlobusMixin):
|
|
|
156
154
|
)
|
|
157
155
|
|
|
158
156
|
|
|
159
|
-
class AssembleQualityData(L1OutputDataBase
|
|
157
|
+
class AssembleQualityData(L1OutputDataBase):
|
|
160
158
|
"""Assemble quality data from the various quality metrics."""
|
|
161
159
|
|
|
162
160
|
def run(self):
|
|
163
161
|
"""Run method for the task."""
|
|
164
|
-
with self.telemetry_span("
|
|
162
|
+
with self.telemetry_span("Computing task type counts"):
|
|
165
163
|
self.compute_and_write_type_counts()
|
|
164
|
+
|
|
165
|
+
with self.telemetry_span("Assembling quality data"):
|
|
166
166
|
quality_data = self.assemble_quality_data()
|
|
167
167
|
|
|
168
168
|
with self.telemetry_span(
|
|
@@ -171,19 +171,14 @@ class AssembleQualityData(L1OutputDataBase, QualityMixin):
|
|
|
171
171
|
self.write(
|
|
172
172
|
quality_data,
|
|
173
173
|
tags=[Tag.output(), Tag.quality_data()],
|
|
174
|
-
encoder=
|
|
174
|
+
encoder=basemodel_encoder,
|
|
175
175
|
relative_path=f"{self.constants.dataset_id}_quality_data.json",
|
|
176
176
|
)
|
|
177
177
|
|
|
178
|
-
def assemble_quality_data(self) -> list[
|
|
178
|
+
def assemble_quality_data(self) -> list[QualityMetric]:
|
|
179
179
|
"""Assemble Quality Data."""
|
|
180
|
-
metrics = []
|
|
181
180
|
tags = [Tag.quality("GENERIC")]
|
|
182
|
-
|
|
183
|
-
# `metrics` needs to be list[dict] for compatibility with existing `quality_assemble_data`
|
|
184
|
-
# once all metric conversion is complete, `metrics` should be list[QualityMetric] instead
|
|
185
|
-
metrics.append(metric.model_dump(mode="json"))
|
|
186
|
-
return metrics
|
|
181
|
+
return list(self.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric))
|
|
187
182
|
|
|
188
183
|
def compute_and_write_type_counts(self) -> None:
|
|
189
184
|
"""Compute and write TASK_TYPES quality metric."""
|
|
@@ -32,11 +32,6 @@ logger = logging.getLogger(__name__)
|
|
|
32
32
|
class QualityMixin:
|
|
33
33
|
"""Mixin class supporting the generation of the quality reports."""
|
|
34
34
|
|
|
35
|
-
@staticmethod
|
|
36
|
-
def _format_warnings(warnings: list[str] | None) -> list[str] | None:
|
|
37
|
-
"""If warnings is an empty list, change its value to None."""
|
|
38
|
-
return warnings or None
|
|
39
|
-
|
|
40
35
|
def _write_quality_metric(self, metric: QualityMetric) -> None:
|
|
41
36
|
"""Write one fully constructed quality metric in the generic format."""
|
|
42
37
|
self.write(metric, tags=Tag.quality("GENERIC"), encoder=basemodel_encoder)
|
|
@@ -414,7 +409,7 @@ class QualityMixin:
|
|
|
414
409
|
metric_code=MetricCode.polcal_global_par_vals,
|
|
415
410
|
facet=label,
|
|
416
411
|
table_data=[TableData(rows=tuple(table_rows))],
|
|
417
|
-
warnings=
|
|
412
|
+
warnings=warnings,
|
|
418
413
|
)
|
|
419
414
|
self._write_quality_metric(metric)
|
|
420
415
|
|
|
@@ -676,7 +671,7 @@ class QualityMixin:
|
|
|
676
671
|
metric_code=MetricCode.polcal_efficiency,
|
|
677
672
|
facet=label,
|
|
678
673
|
efficiency_data=EfficiencyData(efficiency_list=efficiency_list),
|
|
679
|
-
warnings=
|
|
674
|
+
warnings=warnings,
|
|
680
675
|
)
|
|
681
676
|
self._write_quality_metric(metric)
|
|
682
677
|
|
|
@@ -320,10 +320,8 @@ class QualityL0Metrics(WorkflowTaskBase, QualityMixin):
|
|
|
320
320
|
if l0_data.has_values:
|
|
321
321
|
# only compute warnings for the first modstate encountered
|
|
322
322
|
if not warnings_computed:
|
|
323
|
-
warnings = self.
|
|
324
|
-
|
|
325
|
-
datetimes=l0_data.datetimes, values=l0_data.average_values
|
|
326
|
-
)
|
|
323
|
+
warnings = self._find_iqr_outliers(
|
|
324
|
+
datetimes=l0_data.datetimes, values=l0_data.average_values
|
|
327
325
|
)
|
|
328
326
|
warnings_computed = True
|
|
329
327
|
time_series = TimeSeriesData(
|
|
@@ -365,10 +363,8 @@ class QualityL0Metrics(WorkflowTaskBase, QualityMixin):
|
|
|
365
363
|
if l0_data.has_values:
|
|
366
364
|
# only compute warnings for the first modstate encountered
|
|
367
365
|
if not warnings_computed:
|
|
368
|
-
warnings = self.
|
|
369
|
-
|
|
370
|
-
datetimes=l0_data.datetimes, values=l0_data.rms_values
|
|
371
|
-
)
|
|
366
|
+
warnings = self._find_iqr_outliers(
|
|
367
|
+
datetimes=l0_data.datetimes, values=l0_data.rms_values
|
|
372
368
|
)
|
|
373
369
|
warnings_computed = True
|
|
374
370
|
time_series = TimeSeriesData(
|
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
import importlib
|
|
4
4
|
import logging
|
|
5
5
|
from datetime import datetime
|
|
6
|
-
from itertools import chain
|
|
7
6
|
from pathlib import Path
|
|
8
7
|
from typing import Any
|
|
9
8
|
from typing import Generator
|
|
@@ -265,12 +264,32 @@ class CreateTrialQualityReport(OutputDataBase):
|
|
|
265
264
|
def create_trial_quality_report(self) -> None:
|
|
266
265
|
"""Generate a trial quality report in pdf format and save to the file system for future upload."""
|
|
267
266
|
with self.telemetry_span(f"Building the trial quality report"):
|
|
268
|
-
#
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
)
|
|
267
|
+
# the single quality_data file is a list[QualityMetric]
|
|
268
|
+
# it is retrieved here as a list[dict] for compatibility with existing `format_report`
|
|
269
|
+
# Unfortunately `list(read())` adds an unwanted outer list to the single list in the quality_data file
|
|
270
|
+
quality_data_shell = list(
|
|
271
|
+
self.read(tags=Tag.quality_data(), decoder=quality_data_decoder)
|
|
273
272
|
)
|
|
273
|
+
match quality_data_shell:
|
|
274
|
+
# no elements - no quality_data files found
|
|
275
|
+
case []:
|
|
276
|
+
quality_data = []
|
|
277
|
+
# one element - one quality_data file found
|
|
278
|
+
case [quality_data]:
|
|
279
|
+
pass
|
|
280
|
+
# multiple elements - multiple quality_data files found
|
|
281
|
+
case _:
|
|
282
|
+
raise ValueError(
|
|
283
|
+
f"Multiple quality data files not supported. Found {len(quality_data_shell)}."
|
|
284
|
+
)
|
|
285
|
+
# `quality_data` needs to be list[dict] for compatibility with existing `format_report`
|
|
286
|
+
# `format_report` and `quality_data` need to be converted to list[QualityMetric] at the same time
|
|
287
|
+
# future implementation:
|
|
288
|
+
# quality_data_shell: list[list[QualityMetric]] = list(self.read(
|
|
289
|
+
# tags=Tag.quality_data(),
|
|
290
|
+
# decoder=list_of_basemodel_decoder,
|
|
291
|
+
# model=QualityMetric
|
|
292
|
+
# ))
|
|
274
293
|
quality_report = format_report(
|
|
275
294
|
report_data=quality_data, dataset_id=self.constants.dataset_id
|
|
276
295
|
)
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
from datetime import datetime
|
|
2
2
|
from datetime import timedelta
|
|
3
|
-
from itertools import chain
|
|
4
3
|
from pathlib import Path
|
|
5
4
|
from typing import Callable
|
|
6
5
|
from uuid import uuid4
|
|
@@ -662,10 +661,8 @@ def test_assemble_quality_data(
|
|
|
662
661
|
# When
|
|
663
662
|
task()
|
|
664
663
|
# Then
|
|
665
|
-
#
|
|
666
|
-
quality_data =
|
|
667
|
-
chain.from_iterable(task.read(tags=Tag.quality_data(), decoder=json_decoder))
|
|
668
|
-
)
|
|
664
|
+
# the single quality_data file is a list[QualityMetric]
|
|
665
|
+
quality_data = next(task.read(tags=Tag.quality_data(), decoder=json_decoder))
|
|
669
666
|
assert len(quality_data) == 17
|
|
670
667
|
for metric_data in quality_data:
|
|
671
668
|
rm: ReportMetric = ReportMetric.from_dict(metric_data)
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import datetime
|
|
2
1
|
import json
|
|
3
2
|
import os
|
|
4
3
|
from contextlib import nullcontext
|
|
4
|
+
from datetime import datetime
|
|
5
|
+
from datetime import timedelta
|
|
5
6
|
from io import BytesIO
|
|
6
7
|
from io import StringIO
|
|
7
8
|
from math import inf
|
|
@@ -24,13 +25,16 @@ from astropy.io.fits import PrimaryHDU
|
|
|
24
25
|
from astropy.utils.exceptions import AstropyUserWarning
|
|
25
26
|
from pydantic import BaseModel
|
|
26
27
|
from pydantic import Field
|
|
28
|
+
from pydantic import ValidationError
|
|
27
29
|
from pydantic import create_model
|
|
30
|
+
from pydantic_core import PydanticSerializationError
|
|
28
31
|
|
|
29
32
|
from dkist_processing_common.codecs.asdf import asdf_decoder
|
|
30
33
|
from dkist_processing_common.codecs.asdf import asdf_encoder
|
|
31
34
|
from dkist_processing_common.codecs.asdf import asdf_fileobj_encoder
|
|
32
35
|
from dkist_processing_common.codecs.basemodel import basemodel_decoder
|
|
33
36
|
from dkist_processing_common.codecs.basemodel import basemodel_encoder
|
|
37
|
+
from dkist_processing_common.codecs.basemodel import list_of_basemodel_decoder
|
|
34
38
|
from dkist_processing_common.codecs.bytes import bytes_decoder
|
|
35
39
|
from dkist_processing_common.codecs.bytes import bytes_encoder
|
|
36
40
|
from dkist_processing_common.codecs.fits import fits_access_decoder
|
|
@@ -51,11 +55,16 @@ from dkist_processing_common.codecs.quality import quality_data_hook
|
|
|
51
55
|
from dkist_processing_common.codecs.str import str_decoder
|
|
52
56
|
from dkist_processing_common.codecs.str import str_encoder
|
|
53
57
|
from dkist_processing_common.models.fits_access import FitsAccessBase
|
|
58
|
+
from dkist_processing_common.models.quality import TimeSeriesData
|
|
59
|
+
from dkist_processing_common.models.quality import XYData
|
|
54
60
|
|
|
61
|
+
NOW = datetime.now()
|
|
55
62
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
63
|
+
TIME_SERIES_DATA = TimeSeriesData(
|
|
64
|
+
x_values=(NOW, NOW + timedelta(seconds=1), NOW + timedelta(seconds=2)), y_values=(0.0, 0.1, 0.2)
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
SIMPLE_XY_DATA = XYData(xlabel="x", ylabel="y", series_data={"": TIME_SERIES_DATA})
|
|
59
68
|
|
|
60
69
|
|
|
61
70
|
@pytest.fixture
|
|
@@ -118,6 +127,11 @@ def pydantic_basemodel() -> BaseModel:
|
|
|
118
127
|
return Foo(bar=123)
|
|
119
128
|
|
|
120
129
|
|
|
130
|
+
@pytest.fixture
|
|
131
|
+
def list_of_basemodel(pydantic_basemodel) -> list[BaseModel | None]:
|
|
132
|
+
return [pydantic_basemodel, None, pydantic_basemodel]
|
|
133
|
+
|
|
134
|
+
|
|
121
135
|
@pytest.fixture
|
|
122
136
|
def string() -> str:
|
|
123
137
|
return "string"
|
|
@@ -143,6 +157,19 @@ def path_to_text_file(tmp_file) -> Callable[[str], Path]:
|
|
|
143
157
|
return inner
|
|
144
158
|
|
|
145
159
|
|
|
160
|
+
@pytest.fixture
|
|
161
|
+
def write_basemodel_to_file(tmp_file) -> Callable[[BaseModel | list[BaseModel | None]], Path]:
|
|
162
|
+
"""Create a bytes file from a passed BaseModel at runtime and return the Path to the created file"""
|
|
163
|
+
|
|
164
|
+
def inner(basemodel: BaseModel | list[BaseModel | None]) -> Path:
|
|
165
|
+
bytes_data = basemodel_encoder(basemodel)
|
|
166
|
+
with open(tmp_file, "wb") as f:
|
|
167
|
+
f.write(bytes_data)
|
|
168
|
+
return tmp_file
|
|
169
|
+
|
|
170
|
+
return inner
|
|
171
|
+
|
|
172
|
+
|
|
146
173
|
@pytest.fixture
|
|
147
174
|
def asdf_tree() -> dict:
|
|
148
175
|
return {"metadata_value": "something", "data": np.empty((100, 100))}
|
|
@@ -318,7 +345,6 @@ def valid_json_codec(request) -> dict[object, str]:
|
|
|
318
345
|
)
|
|
319
346
|
def valid_quality_codec(
|
|
320
347
|
request,
|
|
321
|
-
now: datetime,
|
|
322
348
|
) -> dict[object, str]:
|
|
323
349
|
"""
|
|
324
350
|
Valid codec transformations for quality_data_encoder and quality_data_decoder.
|
|
@@ -339,8 +365,8 @@ def valid_quality_codec(
|
|
|
339
365
|
"json_str": str(pi),
|
|
340
366
|
},
|
|
341
367
|
"datetime": {
|
|
342
|
-
"python_object":
|
|
343
|
-
"json_str": f'{{"iso_date": "{
|
|
368
|
+
"python_object": NOW,
|
|
369
|
+
"json_str": f'{{"iso_date": "{NOW.isoformat()}"}}',
|
|
344
370
|
},
|
|
345
371
|
"list_of_int": {"python_object": [1, 2, 3], "json_str": "[1, 2, 3]"},
|
|
346
372
|
"list_of_str": {
|
|
@@ -401,7 +427,8 @@ class DummyFitsAccess(FitsAccessBase):
|
|
|
401
427
|
pytest.param("primary_hdu_list", fits_hdulist_encoder, id="fits uncompressed HDUList"),
|
|
402
428
|
pytest.param("compressed_hdu_list", fits_hdulist_encoder, id="fits compressed HDUList"),
|
|
403
429
|
pytest.param("dictionary", json_encoder, id="json"),
|
|
404
|
-
pytest.param("pydantic_basemodel", basemodel_encoder, id="
|
|
430
|
+
pytest.param("pydantic_basemodel", basemodel_encoder, id="pydantic_basemodel"),
|
|
431
|
+
pytest.param("list_of_basemodel", basemodel_encoder, id="list_of_basemodel"),
|
|
405
432
|
pytest.param("string", str_encoder, id="str"),
|
|
406
433
|
pytest.param("asdf_tree", asdf_encoder, id="asdf"),
|
|
407
434
|
pytest.param("asdf_obj", asdf_fileobj_encoder, id="asdf_obj"),
|
|
@@ -786,7 +813,7 @@ def test_json_decoder_valid(valid_json_codec, path_to_text_file):
|
|
|
786
813
|
"python_object, expected_exception_type",
|
|
787
814
|
[
|
|
788
815
|
pytest.param(b"bytes", TypeError, id="bytes"),
|
|
789
|
-
pytest.param(
|
|
816
|
+
pytest.param(NOW, TypeError, id="datetime"),
|
|
790
817
|
],
|
|
791
818
|
)
|
|
792
819
|
def test_json_encoder_invalid(python_object: Any, expected_exception_type: type[Exception]):
|
|
@@ -804,7 +831,150 @@ def test_json_encoder_invalid(python_object: Any, expected_exception_type: type[
|
|
|
804
831
|
json_encoder(python_object)
|
|
805
832
|
|
|
806
833
|
|
|
807
|
-
|
|
834
|
+
@pytest.mark.parametrize(
|
|
835
|
+
"input_str",
|
|
836
|
+
[
|
|
837
|
+
pytest.param("simple_string", id="simple_string"),
|
|
838
|
+
pytest.param("\u2764", id="unicode_heart"),
|
|
839
|
+
],
|
|
840
|
+
)
|
|
841
|
+
def test_basemodel_encoder_valid(input_str: str):
|
|
842
|
+
"""
|
|
843
|
+
Given: a valid pydantic BaseModel object
|
|
844
|
+
When: basemodel encoding is applied to the BaseModel
|
|
845
|
+
Then: the BaseModel get encoded to bytes without error
|
|
846
|
+
"""
|
|
847
|
+
|
|
848
|
+
xy_data = XYData(xlabel="x", ylabel=input_str, series_data={input_str: TIME_SERIES_DATA})
|
|
849
|
+
|
|
850
|
+
output_bytes = basemodel_encoder(xy_data)
|
|
851
|
+
assert isinstance(output_bytes, bytes)
|
|
852
|
+
|
|
853
|
+
|
|
854
|
+
@pytest.mark.parametrize(
|
|
855
|
+
"input_str, expected_exception_type",
|
|
856
|
+
[
|
|
857
|
+
pytest.param("\ud800", PydanticSerializationError, id="invalid_unicode"),
|
|
858
|
+
],
|
|
859
|
+
)
|
|
860
|
+
def test_basemodel_encoder_invalid(input_str: str, expected_exception_type: type[Exception]):
|
|
861
|
+
"""
|
|
862
|
+
Given: an invalid pydantic BaseModel object
|
|
863
|
+
When: basemodel encoding is applied to the BaseModel
|
|
864
|
+
Then: the proper exception gets raised
|
|
865
|
+
"""
|
|
866
|
+
xy_data = XYData(xlabel="x", ylabel=input_str, series_data={input_str: TIME_SERIES_DATA})
|
|
867
|
+
|
|
868
|
+
with pytest.raises(expected_exception_type) as e:
|
|
869
|
+
basemodel_encoder(xy_data)
|
|
870
|
+
|
|
871
|
+
|
|
872
|
+
@pytest.mark.parametrize(
|
|
873
|
+
"basemodel",
|
|
874
|
+
[
|
|
875
|
+
pytest.param(SIMPLE_XY_DATA, id="xy_data"),
|
|
876
|
+
],
|
|
877
|
+
)
|
|
878
|
+
def test_basemodel_decoder_valid(
|
|
879
|
+
basemodel: BaseModel,
|
|
880
|
+
write_basemodel_to_file: Callable[[BaseModel | list[BaseModel | None]], Path],
|
|
881
|
+
):
|
|
882
|
+
"""
|
|
883
|
+
Given: a valid pydantic BaseModel object loaded to the file system as bytes
|
|
884
|
+
When: basemodel decoding is applied to the file
|
|
885
|
+
Then: the retrieved BaseModel is equivalent to the initial BaseModel
|
|
886
|
+
"""
|
|
887
|
+
# Given
|
|
888
|
+
path = write_basemodel_to_file(basemodel)
|
|
889
|
+
model_type = type(basemodel)
|
|
890
|
+
# When
|
|
891
|
+
basemodel_retrieved = basemodel_decoder(path, model=model_type)
|
|
892
|
+
# Then
|
|
893
|
+
assert isinstance(basemodel_retrieved, model_type)
|
|
894
|
+
assert basemodel_retrieved == basemodel
|
|
895
|
+
|
|
896
|
+
|
|
897
|
+
@pytest.mark.parametrize(
|
|
898
|
+
"basemodel, expected_exception_type",
|
|
899
|
+
[
|
|
900
|
+
pytest.param(SIMPLE_XY_DATA, ValidationError, id="is_a_list"),
|
|
901
|
+
],
|
|
902
|
+
)
|
|
903
|
+
def test_basemodel_decoder_invalid(
|
|
904
|
+
basemodel: BaseModel,
|
|
905
|
+
expected_exception_type: type[Exception],
|
|
906
|
+
write_basemodel_to_file: Callable[[BaseModel | list[BaseModel | None]], Path],
|
|
907
|
+
):
|
|
908
|
+
"""
|
|
909
|
+
Given: an invalid BaseModel object loaded to the file system as bytes
|
|
910
|
+
When: basemodel decoding is applied to the file
|
|
911
|
+
Then: the proper exception gets raised
|
|
912
|
+
"""
|
|
913
|
+
# Given
|
|
914
|
+
basemodel_list = [basemodel, None, basemodel]
|
|
915
|
+
path = write_basemodel_to_file(basemodel_list)
|
|
916
|
+
model_type = type(basemodel)
|
|
917
|
+
|
|
918
|
+
# When / Then
|
|
919
|
+
with pytest.raises(expected_exception_type) as e:
|
|
920
|
+
basemodel_decoder(path, model=model_type)
|
|
921
|
+
|
|
922
|
+
|
|
923
|
+
@pytest.mark.parametrize(
|
|
924
|
+
"basemodel",
|
|
925
|
+
[
|
|
926
|
+
pytest.param(SIMPLE_XY_DATA, id="xy_data"),
|
|
927
|
+
],
|
|
928
|
+
)
|
|
929
|
+
def test_list_of_basemodel_decoder_valid(
|
|
930
|
+
basemodel: BaseModel,
|
|
931
|
+
write_basemodel_to_file: Callable[[BaseModel | list[BaseModel | None]], Path],
|
|
932
|
+
):
|
|
933
|
+
"""
|
|
934
|
+
Given: a valid list of pydantic BaseModel objects loaded to the file system as bytes
|
|
935
|
+
When: basemodel decoding is applied to the file
|
|
936
|
+
Then: the retrieved list of BaseModel is equivalent to the initial list of BaseModel
|
|
937
|
+
"""
|
|
938
|
+
# Given
|
|
939
|
+
basemodel_list = [basemodel, None, basemodel]
|
|
940
|
+
path = write_basemodel_to_file(basemodel_list)
|
|
941
|
+
model_type = type(basemodel)
|
|
942
|
+
# When
|
|
943
|
+
list_retrieved = list_of_basemodel_decoder(path, model=model_type)
|
|
944
|
+
# Then
|
|
945
|
+
assert isinstance(list_retrieved, list)
|
|
946
|
+
assert len(list_retrieved) == len(basemodel_list)
|
|
947
|
+
assert isinstance(list_retrieved[0], model_type)
|
|
948
|
+
assert list_retrieved[1] is None
|
|
949
|
+
assert list_retrieved == basemodel_list
|
|
950
|
+
|
|
951
|
+
|
|
952
|
+
@pytest.mark.parametrize(
|
|
953
|
+
"basemodel, expected_exception_type",
|
|
954
|
+
[
|
|
955
|
+
pytest.param(SIMPLE_XY_DATA, ValidationError, id="not_a_list"),
|
|
956
|
+
],
|
|
957
|
+
)
|
|
958
|
+
def test_list_of_basemodel_decoder_invalid(
|
|
959
|
+
basemodel: BaseModel,
|
|
960
|
+
expected_exception_type: type[Exception],
|
|
961
|
+
write_basemodel_to_file: Callable[[BaseModel | list[BaseModel | None]], Path],
|
|
962
|
+
):
|
|
963
|
+
"""
|
|
964
|
+
Given: an invalid list of pydantic BaseModel objects loaded to the file system as bytes
|
|
965
|
+
When: basemodel decoding is applied to the file
|
|
966
|
+
Then: the proper exception gets raised
|
|
967
|
+
"""
|
|
968
|
+
# Given
|
|
969
|
+
path = write_basemodel_to_file(basemodel)
|
|
970
|
+
model_type = type(basemodel)
|
|
971
|
+
|
|
972
|
+
# When / Then
|
|
973
|
+
with pytest.raises(expected_exception_type) as e:
|
|
974
|
+
list_of_basemodel_decoder(path, model=model_type)
|
|
975
|
+
|
|
976
|
+
|
|
977
|
+
def test_basemodel_decoder_json(valid_json_codec, path_to_text_file):
|
|
808
978
|
"""
|
|
809
979
|
Given: a python object that can be validated to a Pydantic BaseModel object is written to file as json
|
|
810
980
|
When: basemodel decoding is applied to the json file
|
|
@@ -20,6 +20,7 @@ from dkist_processing_common.models.quality import QualityMetric
|
|
|
20
20
|
from dkist_processing_common.models.quality import TableData
|
|
21
21
|
from dkist_processing_common.models.quality import XYData
|
|
22
22
|
from dkist_processing_common.models.quality import format_facet
|
|
23
|
+
from dkist_processing_common.models.quality import format_warnings
|
|
23
24
|
from dkist_processing_common.models.tags import Tag
|
|
24
25
|
from dkist_processing_common.models.task_name import TaskName
|
|
25
26
|
from dkist_processing_common.tasks.quality_metrics import L0QualityData
|
|
@@ -1220,3 +1221,21 @@ def test_format_facet(label: str | Any, expected_result: str):
|
|
|
1220
1221
|
Then: the label is properly formatted
|
|
1221
1222
|
"""
|
|
1222
1223
|
assert format_facet(label) == expected_result
|
|
1224
|
+
|
|
1225
|
+
|
|
1226
|
+
@pytest.mark.parametrize(
|
|
1227
|
+
"warnings, expected_result",
|
|
1228
|
+
[
|
|
1229
|
+
pytest.param(None, None, id="none"),
|
|
1230
|
+
pytest.param([], None, id="empty"),
|
|
1231
|
+
pytest.param(["abc"], ["abc"], id="single_element"),
|
|
1232
|
+
pytest.param(["abc", "xyz"], ["abc", "xyz"], id="two_elements"),
|
|
1233
|
+
],
|
|
1234
|
+
)
|
|
1235
|
+
def test_format_warnings(warnings: list[str] | None, expected_result: list[str] | None):
|
|
1236
|
+
"""
|
|
1237
|
+
Given: A list of warnings
|
|
1238
|
+
When: formatting the warnings
|
|
1239
|
+
Then: the warnings are properly formatted
|
|
1240
|
+
"""
|
|
1241
|
+
assert format_warnings(warnings) == expected_result
|
|
@@ -47,17 +47,6 @@ def quality_mixin_task(tmp_path, recipe_run_id) -> QualityMixinTask:
|
|
|
47
47
|
task._purge()
|
|
48
48
|
|
|
49
49
|
|
|
50
|
-
def test_format_warnings(quality_mixin_task):
|
|
51
|
-
"""
|
|
52
|
-
Given: a task with the QualityMixin
|
|
53
|
-
When: checking the format of warnings
|
|
54
|
-
Then: an empty list of warnings is converted to None
|
|
55
|
-
"""
|
|
56
|
-
task = quality_mixin_task
|
|
57
|
-
warnings = task._format_warnings(warnings=[])
|
|
58
|
-
assert warnings is None
|
|
59
|
-
|
|
60
|
-
|
|
61
50
|
@pytest.mark.parametrize(
|
|
62
51
|
"thin, samples_larger_than_total, label",
|
|
63
52
|
[
|
|
@@ -8,8 +8,6 @@ import astropy.units as u
|
|
|
8
8
|
import numpy as np
|
|
9
9
|
import pytest
|
|
10
10
|
from astropy.io import fits
|
|
11
|
-
from dkist_data_simulator.dataset_extras import DatasetExtraBase
|
|
12
|
-
from dkist_data_simulator.dataset_extras import InstrumentTables
|
|
13
11
|
from dkist_data_simulator.spec214.vbi import SimpleVBIDataset
|
|
14
12
|
from sqids import Sqids
|
|
15
13
|
|
|
@@ -19,7 +17,6 @@ from dkist_processing_common.codecs.basemodel import basemodel_encoder
|
|
|
19
17
|
from dkist_processing_common.codecs.bytes import bytes_decoder
|
|
20
18
|
from dkist_processing_common.codecs.fits import fits_hdulist_encoder
|
|
21
19
|
from dkist_processing_common.codecs.json import json_decoder
|
|
22
|
-
from dkist_processing_common.codecs.quality import quality_data_encoder
|
|
23
20
|
from dkist_processing_common.models.input_dataset import InputDatasetFrames
|
|
24
21
|
from dkist_processing_common.models.input_dataset import InputDatasetParameter
|
|
25
22
|
from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
|
|
@@ -319,7 +316,7 @@ def create_trial_quality_report_task(
|
|
|
319
316
|
task.write(
|
|
320
317
|
quality_data_warning_only,
|
|
321
318
|
tags=Tag.quality_data(),
|
|
322
|
-
encoder=
|
|
319
|
+
encoder=basemodel_encoder,
|
|
323
320
|
relative_path=f"{task.constants.dataset_id}_quality_data.json",
|
|
324
321
|
)
|
|
325
322
|
|
|
@@ -8,6 +8,9 @@ bitbucket-pipelines.yml
|
|
|
8
8
|
pyproject.toml
|
|
9
9
|
.rovodev/.review-agent.md
|
|
10
10
|
changelog/.gitempty
|
|
11
|
+
changelog/358.misc.rst
|
|
12
|
+
changelog/363.feature.rst
|
|
13
|
+
changelog/363.misc.rst
|
|
11
14
|
dkist_processing_common/__init__.py
|
|
12
15
|
dkist_processing_common/config.py
|
|
13
16
|
dkist_processing_common/manual.py
|
|
@@ -9,6 +9,9 @@
|
|
|
9
9
|
"README.rst",
|
|
10
10
|
"bitbucket-pipelines.yml",
|
|
11
11
|
"changelog/.gitempty",
|
|
12
|
+
"changelog/358.misc.rst",
|
|
13
|
+
"changelog/363.feature.rst",
|
|
14
|
+
"changelog/363.misc.rst",
|
|
12
15
|
"dkist_processing_common/__init__.py",
|
|
13
16
|
"dkist_processing_common/_util/__init__.py",
|
|
14
17
|
"dkist_processing_common/_util/constants.py",
|