dkist-processing-vbi 2.0.1rc2__tar.gz → 2.1.1__tar.gz

{dkist_processing_vbi-2.0.1rc2 → dkist_processing_vbi-2.1.1}/CHANGELOG.rst

@@ -1,3 +1,52 @@
+v2.1.1 (2026-09-30)
+===================
+
+Misc
+----
+
+- Update dkist-processing-common to 14.5.2 to restore provenance recording for
+  manual parse tasks and trial dataset inventory, ASDF, and quality report tasks. (`#236 <https://bitbucket.org/dkistdc/dkist-processing-vbi/pull-requests/236>`__)
+
+
+v2.1.0 (2026-09-28)
+===================
+
+Features
+--------
+
+- Add the ``l0_to_l1_vbi_bulk`` workflow for bulk processing of L0 data, using the default resource queue for science calibration. (`#233 <https://bitbucket.org/dkistdc/dkist-processing-vbi/pull-requests/233>`__)
+- Enable multi-core parallel science frame calibration in the default VBI pipeline. (`#233 <https://bitbucket.org/dkistdc/dkist-processing-vbi/pull-requests/233>`__)
+
+
+Misc
+----
+
+- Rename workflows ``l0_to_l1_vbi_no-speckle`` to ``l0_to_l1_vbi`` and ``l0_to_l1_vbi_no-speckle-full-trial`` to ``l0_to_l1_vbi_full-trial``. (`#233 <https://bitbucket.org/dkistdc/dkist-processing-vbi/pull-requests/233>`__)
+
+
+v2.0.1 (2026-09-24)
+===================
+
+Bugfixes
+--------
+
+- Fix the typo in the name for the "VBI-Red Fe XI" filter. (`#235 <https://bitbucket.org/dkistdc/dkist-processing-vbi/pull-requests/235>`__)
+
+
+Misc
+----
+
+- Use the "level0_spec214" file schema keyword argument in test Datasets.
+  With this keyword, subsequent translation of the resulting headers to
+  Spec 214 is not necessary. (`#223 <https://bitbucket.org/dkistdc/dkist-processing-vbi/pull-requests/223>`__)
+- Use dkist-fits-specifications v4.29.0, which includes the Trend FITS specification.
+  Use dkist-header-validator v6.0.0, a significant refactor that allows multi-HDU objects to be validated.
+  Use dkist-processing-common v14.5.0, which accommodates the changes to the header validator. (`#223 <https://bitbucket.org/dkistdc/dkist-processing-vbi/pull-requests/223>`__)
+- Validators no longer have validate-and-translate methods.
+  Where still necessary, replace validate-and-translate calls with a call to
+  validate and then a call to translate. (`#223 <https://bitbucket.org/dkistdc/dkist-processing-vbi/pull-requests/223>`__)
+
+
 v2.0.0 (2026-09-03)
 ===================
 

{dkist_processing_vbi-2.0.1rc2 → dkist_processing_vbi-2.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-vbi
-Version: 2.0.1rc2
+Version: 2.1.1
 Summary: Processing code for the VBI instrument on DKIST
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -13,10 +13,10 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.13
 Description-Content-Type: text/x-rst
-Requires-Dist: dkist-processing-common==14.5.0rc3
+Requires-Dist: dkist-processing-common==14.5.2
 Requires-Dist: dkist-processing-math==2.2.3
-Requires-Dist: dkist-header-validator==6.0.0rc1
-Requires-Dist: dkist-fits-specifications==4.29.0rc7
+Requires-Dist: dkist-header-validator==6.0.0
+Requires-Dist: dkist-fits-specifications==4.29.0
 Requires-Dist: astropy==7.0.2
 Requires-Dist: numpy==2.2.5
 Requires-Dist: sunpy==6.1.1
@@ -55,19 +55,19 @@ Requires-Dist: sphinx-autoapi; extra == "docs"
 Requires-Dist: pytest; extra == "docs"
 Requires-Dist: dkist-sphinx-theme; extra == "docs"
 Provides-Extra: frozen
-Requires-Dist: Deprecated==1.3.1; extra == "frozen"
-Requires-Dist: ImageIO==2.37.4; extra == "frozen"
+Requires-Dist: Deprecated==3.0.0; extra == "frozen"
+Requires-Dist: ImageIO==2.38.0; extra == "frozen"
 Requires-Dist: Jinja2==3.1.6; extra == "frozen"
 Requires-Dist: Mako==1.4.3; extra == "frozen"
 Requires-Dist: MarkupSafe==3.0.3; extra == "frozen"
 Requires-Dist: PIMS==0.7; extra == "frozen"
 Requires-Dist: PeakUtils==1.3.5; extra == "frozen"
 Requires-Dist: PyAVM==0.9.9; extra == "frozen"
-Requires-Dist: PyJWT==2.15.0; extra == "frozen"
+Requires-Dist: PyJWT==2.15.1; extra == "frozen"
 Requires-Dist: PyYAML==6.0.3; extra == "frozen"
 Requires-Dist: PyYAML-ft==8.0.0; extra == "frozen"
 Requires-Dist: Pygments==2.21.0; extra == "frozen"
-Requires-Dist: SQLAlchemy==2.0.54; extra == "frozen"
+Requires-Dist: SQLAlchemy==2.1.1; extra == "frozen"
 Requires-Dist: Send2Trash==2.1.0; extra == "frozen"
 Requires-Dist: a2wsgi==1.10.10; extra == "frozen"
 Requires-Dist: aioftp==0.28.3; extra == "frozen"
@@ -83,14 +83,15 @@ Requires-Dist: annotated-types==0.8.0; extra == "frozen"
 Requires-Dist: anyio==4.15.1; extra == "frozen"
 Requires-Dist: apache-airflow==3.3.1; extra == "frozen"
 Requires-Dist: apache-airflow-core==3.3.1; extra == "frozen"
-Requires-Dist: apache-airflow-providers-celery==3.24.0; extra == "frozen"
-Requires-Dist: apache-airflow-providers-common-compat==1.19.0; extra == "frozen"
-Requires-Dist: apache-airflow-providers-common-io==1.9.0; extra == "frozen"
-Requires-Dist: apache-airflow-providers-common-sql==2.1.1; extra == "frozen"
-Requires-Dist: apache-airflow-providers-postgres==7.0.2; extra == "frozen"
-Requires-Dist: apache-airflow-providers-smtp==3.0.4; extra == "frozen"
-Requires-Dist: apache-airflow-providers-standard==1.19.0; extra == "frozen"
+Requires-Dist: apache-airflow-providers-celery==3.24.1; extra == "frozen"
+Requires-Dist: apache-airflow-providers-common-compat==1.20.0; extra == "frozen"
+Requires-Dist: apache-airflow-providers-common-io==1.10.0; extra == "frozen"
+Requires-Dist: apache-airflow-providers-common-sql==2.2.0; extra == "frozen"
+Requires-Dist: apache-airflow-providers-postgres==7.1.0; extra == "frozen"
+Requires-Dist: apache-airflow-providers-smtp==3.1.0; extra == "frozen"
+Requires-Dist: apache-airflow-providers-standard==1.20.0; extra == "frozen"
 Requires-Dist: apache-airflow-task-sdk==1.3.1; extra == "frozen"
+Requires-Dist: appnope==1.0.0; extra == "frozen"
 Requires-Dist: argcomplete==3.7.2; extra == "frozen"
 Requires-Dist: argon2-cffi==25.1.0; extra == "frozen"
 Requires-Dist: argon2-cffi-bindings==26.1.0; extra == "frozen"
@@ -100,7 +101,7 @@ Requires-Dist: asdf_standard==1.5.0; extra == "frozen"
 Requires-Dist: asgiref==3.12.1; extra == "frozen"
 Requires-Dist: asteval==1.0.10; extra == "frozen"
 Requires-Dist: astropy==7.0.2; extra == "frozen"
-Requires-Dist: astropy-iers-data==0.2026.9.21.0.56.25; extra == "frozen"
+Requires-Dist: astropy-iers-data==0.2026.9.28.0.59.37; extra == "frozen"
 Requires-Dist: astropy_healpix==2.0.1; extra == "frozen"
 Requires-Dist: asttokens==3.0.2; extra == "frozen"
 Requires-Dist: async-lru==2.3.0; extra == "frozen"
@@ -110,14 +111,14 @@ Requires-Dist: babel==2.18.0; extra == "frozen"
 Requires-Dist: beautifulsoup4==4.15.0; extra == "frozen"
 Requires-Dist: billiard==4.3.0; extra == "frozen"
 Requires-Dist: bleach==6.4.0; extra == "frozen"
-Requires-Dist: boto3==1.43.101; extra == "frozen"
-Requires-Dist: botocore==1.43.101; extra == "frozen"
+Requires-Dist: boto3==1.43.106; extra == "frozen"
+Requires-Dist: botocore==1.43.106; extra == "frozen"
 Requires-Dist: cachetools==7.2.0; extra == "frozen"
 Requires-Dist: cadwyn==7.0.0; extra == "frozen"
 Requires-Dist: celery==5.6.3; extra == "frozen"
 Requires-Dist: certifi==2026.7.22; extra == "frozen"
 Requires-Dist: cffi==2.1.1; extra == "frozen"
-Requires-Dist: charset-normalizer==3.5.1; extra == "frozen"
+Requires-Dist: charset-normalizer==3.5.2; extra == "frozen"
 Requires-Dist: click==8.5.0; extra == "frozen"
 Requires-Dist: click-didyoumean==0.3.1; extra == "frozen"
 Requires-Dist: click-plugins==1.1.1.2; extra == "frozen"
@@ -126,9 +127,9 @@ Requires-Dist: cloudpickle==3.1.2; extra == "frozen"
 Requires-Dist: colorlog==6.12.0; extra == "frozen"
 Requires-Dist: comm==0.2.3; extra == "frozen"
 Requires-Dist: contourpy==1.4.0; extra == "frozen"
-Requires-Dist: cron_descriptor==2.1.0; extra == "frozen"
+Requires-Dist: cron_descriptor==2.1.1; extra == "frozen"
 Requires-Dist: croniter==6.2.4; extra == "frozen"
-Requires-Dist: cryptography==50.0.1; extra == "frozen"
+Requires-Dist: cryptography==50.0.2; extra == "frozen"
 Requires-Dist: cycler==0.12.1; extra == "frozen"
 Requires-Dist: dask==2026.8.0; extra == "frozen"
 Requires-Dist: dask-image==2026.5.0; extra == "frozen"
@@ -136,15 +137,15 @@ Requires-Dist: debugpy==1.8.22; extra == "frozen"
 Requires-Dist: decorator==5.3.1; extra == "frozen"
 Requires-Dist: defusedxml==0.7.1; extra == "frozen"
 Requires-Dist: dill==0.4.1; extra == "frozen"
-Requires-Dist: dkist-header-validator==6.0.0rc1; extra == "frozen"
-Requires-Dist: dkist-processing-common==14.5.0rc3; extra == "frozen"
+Requires-Dist: dkist-header-validator==6.0.0; extra == "frozen"
+Requires-Dist: dkist-processing-common==14.5.2; extra == "frozen"
 Requires-Dist: dkist-processing-core==7.5.0; extra == "frozen"
 Requires-Dist: dkist-processing-math==2.2.3; extra == "frozen"
 Requires-Dist: dkist-processing-pac==3.1.2; extra == "frozen"
-Requires-Dist: dkist-processing-vbi==2.0.1rc2; extra == "frozen"
+Requires-Dist: dkist-processing-vbi==2.1.1; extra == "frozen"
 Requires-Dist: dkist-service-configuration==4.3.0; extra == "frozen"
 Requires-Dist: dkist-spectral-lines==3.0.1; extra == "frozen"
-Requires-Dist: dkist_fits_specifications==4.29.0rc7; extra == "frozen"
+Requires-Dist: dkist_fits_specifications==4.29.0; extra == "frozen"
 Requires-Dist: dnspython==2.8.0; extra == "frozen"
 Requires-Dist: donfig==0.8.1.post1; extra == "frozen"
 Requires-Dist: email-validator==2.3.0; extra == "frozen"
@@ -153,13 +154,13 @@ Requires-Dist: fastapi==0.136.3; extra == "frozen"
 Requires-Dist: fastapi-cli==0.0.32; extra == "frozen"
 Requires-Dist: fastjsonschema==2.22.2; extra == "frozen"
 Requires-Dist: flower==2.2.0; extra == "frozen"
-Requires-Dist: fonttools==4.66.0; extra == "frozen"
-Requires-Dist: fqdn==1.5.1; extra == "frozen"
+Requires-Dist: fonttools==4.66.1; extra == "frozen"
+Requires-Dist: fqdn==1.6.0; extra == "frozen"
 Requires-Dist: frozenlist==1.8.0; extra == "frozen"
 Requires-Dist: fsspec==2026.9.0; extra == "frozen"
-Requires-Dist: globus-sdk==4.9.0; extra == "frozen"
-Requires-Dist: google-crc32c==1.8.0; extra == "frozen"
-Requires-Dist: googleapis-common-protos==1.75.3; extra == "frozen"
+Requires-Dist: globus-sdk==4.9.1; extra == "frozen"
+Requires-Dist: google-crc32c==1.9.0; extra == "frozen"
+Requires-Dist: googleapis-common-protos==1.75.5; extra == "frozen"
 Requires-Dist: gqlclient==2.0.0; extra == "frozen"
 Requires-Dist: greenback==1.3.0; extra == "frozen"
 Requires-Dist: greenlet==3.5.6; extra == "frozen"
@@ -172,7 +173,7 @@ Requires-Dist: humanize==4.16.0; extra == "frozen"
 Requires-Dist: idna==3.20; extra == "frozen"
 Requires-Dist: imageio-ffmpeg==0.6.0; extra == "frozen"
 Requires-Dist: importlib_metadata==9.0.1; extra == "frozen"
-Requires-Dist: ipykernel==7.3.0; extra == "frozen"
+Requires-Dist: ipykernel==7.4.0; extra == "frozen"
 Requires-Dist: ipython==9.17.1; extra == "frozen"
 Requires-Dist: ipython_pygments_lexers==1.1.1; extra == "frozen"
 Requires-Dist: isoduration==20.11.0; extra == "frozen"
@@ -212,7 +213,7 @@ Requires-Dist: methodtools==0.4.7; extra == "frozen"
 Requires-Dist: mistune==3.3.4; extra == "frozen"
 Requires-Dist: more-itertools==11.1.0; extra == "frozen"
 Requires-Dist: moviepy==2.2.1; extra == "frozen"
-Requires-Dist: msgspec==0.21.1; extra == "frozen"
+Requires-Dist: msgspec==0.22.0; extra == "frozen"
 Requires-Dist: multidict==6.9.1; extra == "frozen"
 Requires-Dist: natsort==8.4.0; extra == "frozen"
 Requires-Dist: nbclient==0.11.0; extra == "frozen"
@@ -225,30 +226,32 @@ Requires-Dist: notebook_shim==0.2.4; extra == "frozen"
 Requires-Dist: numcodecs==0.17.0; extra == "frozen"
 Requires-Dist: numpy==2.2.5; extra == "frozen"
 Requires-Dist: object-clerk==1.0.0; extra == "frozen"
-Requires-Dist: opentelemetry-api==1.44.0; extra == "frozen"
-Requires-Dist: opentelemetry-exporter-otlp==1.44.0; extra == "frozen"
-Requires-Dist: opentelemetry-exporter-otlp-proto-common==1.44.0; extra == "frozen"
-Requires-Dist: opentelemetry-exporter-otlp-proto-grpc==1.44.0; extra == "frozen"
-Requires-Dist: opentelemetry-exporter-otlp-proto-http==1.44.0; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation==0.65b0; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-aiohttp-client==0.65b0; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-asgi==0.65b0; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-botocore==0.65b0; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-celery==0.65b0; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-dbapi==0.65b0; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-fastapi==0.65b0; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-pika==0.65b0; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-psycopg2==0.65b0; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-pymongo==0.65b0; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-redis==0.65b0; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-requests==0.65b0; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-sqlalchemy==0.65b0; extra == "frozen"
-Requires-Dist: opentelemetry-instrumentation-system-metrics==0.65b0; extra == "frozen"
+Requires-Dist: opentelemetry-api==1.45.0; extra == "frozen"
+Requires-Dist: opentelemetry-exporter-http-transport==0.66b0; extra == "frozen"
+Requires-Dist: opentelemetry-exporter-otlp==1.45.0; extra == "frozen"
+Requires-Dist: opentelemetry-exporter-otlp-common==0.66b0; extra == "frozen"
+Requires-Dist: opentelemetry-exporter-otlp-proto-common==1.45.0; extra == "frozen"
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc==1.45.0; extra == "frozen"
+Requires-Dist: opentelemetry-exporter-otlp-proto-http==1.45.0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation==0.66b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-aiohttp-client==0.66b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-asgi==0.66b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-botocore==0.66b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-celery==0.66b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-dbapi==0.66b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-fastapi==0.66b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-pika==0.66b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-psycopg2==0.66b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-pymongo==0.66b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-redis==0.66b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-requests==0.66b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-sqlalchemy==0.66b0; extra == "frozen"
+Requires-Dist: opentelemetry-instrumentation-system-metrics==0.66b0; extra == "frozen"
 Requires-Dist: opentelemetry-propagator-aws-xray==1.0.2; extra == "frozen"
-Requires-Dist: opentelemetry-proto==1.44.0; extra == "frozen"
-Requires-Dist: opentelemetry-sdk==1.44.0; extra == "frozen"
-Requires-Dist: opentelemetry-semantic-conventions==0.65b0; extra == "frozen"
-Requires-Dist: opentelemetry-util-http==0.65b0; extra == "frozen"
+Requires-Dist: opentelemetry-proto==1.45.0; extra == "frozen"
+Requires-Dist: opentelemetry-sdk==1.45.0; extra == "frozen"
+Requires-Dist: opentelemetry-semantic-conventions==0.66b0; extra == "frozen"
+Requires-Dist: opentelemetry-util-http==0.66b0; extra == "frozen"
 Requires-Dist: outcome==1.3.0.post0; extra == "frozen"
 Requires-Dist: packaging==26.3; extra == "frozen"
 Requires-Dist: pandas==3.0.6; extra == "frozen"
@@ -263,7 +266,7 @@ Requires-Dist: pexpect==4.9.0; extra == "frozen"
 Requires-Dist: pika==1.4.4; extra == "frozen"
 Requires-Dist: pillow==10.4.0; extra == "frozen"
 Requires-Dist: pip==26.2.1; extra == "frozen"
-Requires-Dist: platformdirs==4.11.12; extra == "frozen"
+Requires-Dist: platformdirs==4.12.2; extra == "frozen"
 Requires-Dist: pluggy==1.6.0; extra == "frozen"
 Requires-Dist: pooch==1.9.0; extra == "frozen"
 Requires-Dist: proglog==0.1.12; extra == "frozen"
@@ -290,8 +293,8 @@ Requires-Dist: python-dateutil==2.9.0.post0; extra == "frozen"
 Requires-Dist: python-dotenv==1.2.3; extra == "frozen"
 Requires-Dist: python-json-logger==4.2.0; extra == "frozen"
 Requires-Dist: python-multipart==0.0.32; extra == "frozen"
-Requires-Dist: python-slugify==9.1.1; extra == "frozen"
-Requires-Dist: pytz==2026.3.post1; extra == "frozen"
+Requires-Dist: python-slugify==9.1.2; extra == "frozen"
+Requires-Dist: pytz==2026.4; extra == "frozen"
 Requires-Dist: pyzmq==27.2.0; extra == "frozen"
 Requires-Dist: redis==6.4.0; extra == "frozen"
 Requires-Dist: referencing==0.37.0; extra == "frozen"
@@ -314,7 +317,7 @@ Requires-Dist: six==1.17.0; extra == "frozen"
 Requires-Dist: slicerator==1.1.0; extra == "frozen"
 Requires-Dist: sniffio==1.3.1; extra == "frozen"
 Requires-Dist: solar-wavelength-calibration==2.0.3; extra == "frozen"
-Requires-Dist: soupsieve==2.9.2; extra == "frozen"
+Requires-Dist: soupsieve==2.10; extra == "frozen"
 Requires-Dist: sqids==0.5.1; extra == "frozen"
 Requires-Dist: sqlparse==0.6.0; extra == "frozen"
 Requires-Dist: stack-data==0.6.3; extra == "frozen"
@@ -344,7 +347,7 @@ Requires-Dist: universal_pathlib==0.3.10; extra == "frozen"
 Requires-Dist: uri-template==1.3.0; extra == "frozen"
 Requires-Dist: urllib3==2.8.0; extra == "frozen"
 Requires-Dist: uuid6==2025.0.1; extra == "frozen"
-Requires-Dist: uvicorn==0.53.0; extra == "frozen"
+Requires-Dist: uvicorn==0.54.0; extra == "frozen"
 Requires-Dist: uvloop==0.22.1; extra == "frozen"
 Requires-Dist: vine==5.1.0; extra == "frozen"
 Requires-Dist: voluptuous==0.16.0; extra == "frozen"
@@ -355,7 +358,7 @@ Requires-Dist: webencodings==0.6.1; extra == "frozen"
 Requires-Dist: websocket-client==1.9.2; extra == "frozen"
 Requires-Dist: websockets==17.1; extra == "frozen"
 Requires-Dist: wirerope==1.0.0; extra == "frozen"
-Requires-Dist: wrapt==2.4.1; extra == "frozen"
+Requires-Dist: wrapt==2.5.0; extra == "frozen"
 Requires-Dist: yamale==6.1.0; extra == "frozen"
 Requires-Dist: yarl==1.25.1; extra == "frozen"
 Requires-Dist: zarr==3.4.0; extra == "frozen"

{dkist_processing_vbi-2.0.1rc2 → dkist_processing_vbi-2.1.1}/dkist_processing_vbi/models/filter.py

@@ -25,7 +25,7 @@ VBI_FILTERS = [
     Filter(name="VBI-Red H-alpha", min=656.258 * u.nm, max=656.306 * u.nm),
     Filter(name="VBI-Red Continuum", min=668.202 * u.nm, max=668.644 * u.nm),
     Filter(name="VBI-Red Ti O", min=705.545 * u.nm, max=706.133 * u.nm),
-    Filter(name="VBI-Red Fe IX", min=789.168 * u.nm, max=789.204 * u.nm),
+    Filter(name="VBI-Red Fe XI", min=789.168 * u.nm, max=789.204 * u.nm),
 ]
 
 

dkist_processing_vbi-2.1.1/dkist_processing_vbi/tasks/science.py

@@ -0,0 +1,298 @@
+"""VBI science task."""
+
+from concurrent.futures import FIRST_COMPLETED
+from concurrent.futures import ProcessPoolExecutor
+from concurrent.futures import wait
+from functools import lru_cache
+from functools import partial
+from itertools import islice
+from pathlib import Path
+
+import numpy as np
+from astropy.io import fits
+from dkist_processing_common.codecs.fits import fits_access_decoder
+from dkist_processing_common.codecs.fits import fits_array_decoder
+from dkist_processing_common.codecs.fits import fits_hdulist_encoder
+from dkist_processing_common.models.task_name import TaskName
+from dkist_processing_common.tasks.mixin.quality import QualityMixin
+from dkist_processing_math.arithmetic import divide_fits_access_by_array
+from dkist_processing_math.arithmetic import subtract_array_from_fits_access
+from dkist_service_configuration.logging import logger
+
+from dkist_processing_vbi.config import dkist_processing_vbi_configurations
+from dkist_processing_vbi.models.tags import VbiTag
+from dkist_processing_vbi.parsers.vbi_l0_fits_access import VbiL0FitsAccess
+from dkist_processing_vbi.tasks.vbi_base import VbiTaskBase
+
+__all__ = ["ScienceCalibration"]
+
+
+# One entry per calibration type (dark, gain, bad-pixel map).
+@lru_cache(maxsize=3)
+def _mmap_calibration_array(path: Path) -> np.ndarray:
+    """Memory-map and cache a calibration array in the current worker process."""
+    array = fits_array_decoder(
+        path,
+        auto_squeeze=False,
+        checksum=False,
+        memmap=True,
+    )
+    array.flags.writeable = False
+    return array
+
+
+def _decode_calibration_file(path: Path) -> None:
+    """Decode a calibration FITS file with checksum checking and discard the array."""
+    array = fits_array_decoder(
+        path,
+        auto_squeeze=False,
+        checksum=True,
+        memmap=True,
+    )
+    del array
+
+
+class ScienceCalibration(VbiTaskBase, QualityMixin):
+    """
+    Task class for running full science calibration on a set of observe images.
+
+    Parameters
+    ----------
+    recipe_run_id : int
+        id of the recipe run used to identify the workflow run this task is part of
+    workflow_name : str
+        name of the workflow to which this instance of the task belongs
+    workflow_version : str
+        version of the workflow to which this instance of the task belongs
+    """
+
+    record_provenance = True
+
+    @staticmethod
+    def correct_bad_pixels(
+        image: np.ndarray,
+        bad_pixel_map: np.ndarray,
+        neighborhood_radius: int,
+    ) -> np.ndarray:
+        """Replace bad pixels with a masked neighborhood median."""
+        bad_pixel_map = np.squeeze(bad_pixel_map).astype(bool)
+
+        corrected_image = image.copy()
+        bad_rows, bad_columns = np.where(bad_pixel_map)
+        source_plane = image[0]
+        corrected_plane = corrected_image[0]
+        masked_source_plane = np.ma.masked_array(
+            source_plane,
+            mask=bad_pixel_map,
+        )
+        for row, column in zip(bad_rows, bad_columns, strict=True):
+            row_slice = slice(
+                max(0, row - neighborhood_radius),
+                min(source_plane.shape[0], row + neighborhood_radius + 1),
+            )
+            column_slice = slice(
+                max(0, column - neighborhood_radius),
+                min(source_plane.shape[1], column + neighborhood_radius + 1),
+            )
+            corrected_plane[row, column] = np.ma.median(
+                masked_source_plane[row_slice, column_slice]
+            )
+
+        return corrected_image
+
+    @staticmethod
+    def calibrate_science_frame(
+        science_access: VbiL0FitsAccess,
+        dark_calibration_array: np.ndarray,
+        gain_calibration_array: np.ndarray,
+        bad_pixel_map: np.ndarray,
+        bad_pixel_neighborhood_radius: int,
+    ) -> VbiL0FitsAccess:
+        """Calibrate one decoded science frame in a worker process."""
+        science_access = next(
+            subtract_array_from_fits_access(
+                access_objs=science_access,
+                array_to_subtract=dark_calibration_array,
+            )
+        )
+        science_access = next(
+            divide_fits_access_by_array(
+                access_objs=science_access,
+                array_to_divide_by=gain_calibration_array,
+            )
+        )
+        science_access.data = ScienceCalibration.correct_bad_pixels(
+            image=science_access.data,
+            bad_pixel_map=bad_pixel_map,
+            neighborhood_radius=bad_pixel_neighborhood_radius,
+        )
+        return science_access
+
+    @staticmethod
+    def encode_calibrated_science_frame(
+        science_access: VbiL0FitsAccess,
+        mosaic: int,
+        num_mosaic_repeats: int,
+    ) -> bytes:
+        """Encode one calibrated science frame as compressed FITS bytes."""
+        # Run the header through an `ImageHDU` to strip out any `PrimaryHDU`-only keys. This is to get around an
+        # `astropy` bug (since 7.0) that produces bad headers if a `CompImageHDU` is populated with a header
+        # from a `PrimaryHDU`.
+        cleaned_header = fits.ImageHDU(header=science_access.header).header
+        processed_hdu_list = fits.HDUList(
+            [fits.PrimaryHDU(), fits.CompImageHDU(data=science_access.data, header=cleaned_header)]
+        )
+        processed_hdu_list[1].header["VBINMOSC"] = num_mosaic_repeats
+        processed_hdu_list[1].header["VBICMOSC"] = mosaic
+
+        return fits_hdulist_encoder(processed_hdu_list)
+
+    @staticmethod
+    def calibrate_and_encode_science_frame(
+        science_path: Path,
+        dark_calibration_path: Path,
+        gain_calibration_path: Path,
+        bad_pixel_map_path: Path,
+        bad_pixel_neighborhood_radius: int,
+        mosaic: int,
+        num_mosaic_repeats: int,
+    ) -> tuple[bytes, int]:
+        """Decode, calibrate, and FITS-encode one science frame, returning bytes and its exposure number."""
+        fits_obj = fits_access_decoder(
+            science_path,
+            fits_access_class=VbiL0FitsAccess,
+            auto_squeeze=False,
+        )
+        fits_obj = ScienceCalibration.calibrate_science_frame(
+            science_access=fits_obj,
+            dark_calibration_array=_mmap_calibration_array(dark_calibration_path),
+            gain_calibration_array=_mmap_calibration_array(gain_calibration_path),
+            bad_pixel_map=_mmap_calibration_array(bad_pixel_map_path),
+            bad_pixel_neighborhood_radius=bad_pixel_neighborhood_radius,
+        )
+        encoded_frame = ScienceCalibration.encode_calibrated_science_frame(
+            science_access=fits_obj,
+            mosaic=mosaic,
+            num_mosaic_repeats=num_mosaic_repeats,
+        )
+        return encoded_frame, fits_obj.current_mosaic_step_exp
+
+    def run(self) -> None:
+        """
+        Perform science calibrations.
+
+        - Do initial array corrections (dark, gain, bad pixels)
+        - Write out L1 science frames
+        - Record quality metrics
+
+        Returns
+        -------
+        None
+
+        """
+        logger.info(
+            f"Starting science with {self.constants.num_spatial_steps} steps and {self.constants.num_mosaic_repeats} mosaic repeats"
+        )
+        worker_count = dkist_processing_vbi_configurations.cpu_count
+        logger.info(f"Calibrating science frames with {worker_count} worker processes")
+        with (
+            self.telemetry_span(
+                f"Reducing science frames from {self.constants.num_spatial_steps} steps and {self.constants.num_mosaic_repeats} mosaic repeats",
+            ),
+            ProcessPoolExecutor(max_workers=worker_count) as executor,
+        ):
+            for step in range(1, self.constants.num_spatial_steps + 1):
+                logger.info(f"retrieving gain calibration for step {step}")
+                gain_tags = [
+                    VbiTag.intermediate(),
+                    VbiTag.task_gain_frame(spatial_step=step),
+                ]
+                gain_calibration_path = next(self.read(tags=gain_tags))
+                _decode_calibration_file(gain_calibration_path)
+
+                logger.info(f"retrieving bad pixel map for step {step}")
+                bad_pixel_map_tags = [
+                    VbiTag.intermediate(),
+                    VbiTag.bad_pixel_map(spatial_step=step),
+                ]
+                bad_pixel_map_path = next(self.read(tags=bad_pixel_map_tags))
+                _decode_calibration_file(bad_pixel_map_path)
+
+                for exp_time in self.constants.observe_exposure_times:
+                    logger.info(f"retrieving dark calibration for step {step} and {exp_time = }")
+                    dark_tags = [
+                        VbiTag.intermediate(),
+                        VbiTag.task_dark_frame(spatial_step=step, exposure_time=exp_time),
+                    ]
+                    dark_calibration_path = next(self.read(tags=dark_tags))
+                    _decode_calibration_file(dark_calibration_path)
+
+                    for mosaic in range(1, self.constants.num_mosaic_repeats + 1):
+                        apm_str = f"step {step} and repeat number {mosaic}"
+                        logger.info(f"collecting observe frames for {apm_str}")
+                        science_paths = self.read(
+                            tags=[
+                                VbiTag.input(),
+                                VbiTag.frame(),
+                                VbiTag.task_observe(),
+                                VbiTag.mosaic(mosaic),
+                                VbiTag.spatial_step(step),
+                                VbiTag.exposure_time(exp_time),
+                            ],
+                        )
+
+                        calibrate_frame = partial(
+                            self.calibrate_and_encode_science_frame,
+                            dark_calibration_path=dark_calibration_path,
+                            gain_calibration_path=gain_calibration_path,
+                            bad_pixel_map_path=bad_pixel_map_path,
+                            bad_pixel_neighborhood_radius=self.parameters.bad_pixel_neighborhood_radius,
+                            mosaic=mosaic,
+                            num_mosaic_repeats=self.constants.num_mosaic_repeats,
+                        )
+                        # Keep one extra frame per worker queued while results are written.
+                        pending_frames = {
+                            executor.submit(calibrate_frame, science_path)
+                            for science_path in islice(science_paths, 2 * worker_count)
+                        }
+                        while pending_frames:
+                            completed_frames, pending_frames = wait(
+                                pending_frames, return_when=FIRST_COMPLETED
+                            )
+                            while completed_frames:
+                                encoded_frame, exp_num = completed_frames.pop().result()
+                                logger.info(f"Writing output for {apm_str} and {exp_num = }")
+                                self.write_calibrated_fits_bytes(encoded_frame, mosaic, step)
+                                del encoded_frame
+
+                                science_path = next(science_paths, None)
+                                if science_path is not None:
+                                    pending_frames.add(
+                                        executor.submit(calibrate_frame, science_path)
+                                    )
+
+        with self.telemetry_span("Computing and logging quality metrics"):
+            no_of_raw_obs_frames: int = self.count(
+                tags=[
+                    VbiTag.input(),
+                    VbiTag.frame(),
+                    VbiTag.task_observe(),
+                ],
+            )
+            self.quality_write_intermediate_task_type_counts(
+                task_type=TaskName.observe.value, total_frames=no_of_raw_obs_frames
+            )
+
+    def write_calibrated_fits_bytes(self, encoded_frame: bytes, mosaic: int, step: int) -> None:
+        """Write encoded calibrated FITS bytes to scratch and tag them correctly."""
+        # It is an intentional decision to not tag with exposure time here
+        self.write(
+            data=encoded_frame,
+            tags=[
+                VbiTag.calibrated(),
+                VbiTag.frame(),
+                VbiTag.spatial_step(step),
+                VbiTag.mosaic(mosaic),
+                VbiTag.stokes("I"),
+            ],
+        )

{dkist_processing_vbi-2.0.1rc2 → dkist_processing_vbi-2.1.1}/dkist_processing_vbi/tests/test_science.py

@@ -143,15 +143,21 @@ def science_calibration_task(
     "abort_mosaic",
     [pytest.param(False, id="Full set"), pytest.param(True, id="Aborted last mosaic")],
 )
-def test_science_calibration(science_calibration_task, mocker, fake_gql_client):
+@pytest.mark.parametrize("cpu_count", [1, 2], ids=["single-core", "multi-core"])
+def test_science_calibration(science_calibration_task, mocker, fake_gql_client, cpu_count):
     """
     Given: a set of parsed input observe frames, dark and gain calibrations, and a ScienceCalibration task
-    When: the task is run
+    When: the task is run with one or two worker processes
     Then: the science frames are processed, no exposures are averaged, and the array values are correct
     """
     mocker.patch(
         "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=fake_gql_client
     )
+    mocker.patch(
+        "dkist_processing_vbi.config.DKISTProcessingVBIConfigurations.cpu_count",
+        new_callable=mocker.PropertyMock,
+        return_value=cpu_count,
+    )
     task, num_steps, num_exp_per_step, expected_num_mosaic_repeats = science_calibration_task
     task()
 
@@ -213,6 +219,7 @@ def test_correct_bad_pixels(science_calibration_task):
     corrected_image = task.correct_bad_pixels(
         image=image,
         bad_pixel_map=bad_pixel_map,
+        neighborhood_radius=task.parameters.bad_pixel_neighborhood_radius,
     )
 
     neighborhood = image[0, :11, :11]
@@ -241,6 +248,7 @@ def test_correct_clustered_bad_pixels(science_calibration_task, abort_mosaic):
     corrected_image = task.correct_bad_pixels(
         image=image,
         bad_pixel_map=bad_pixel_map,
+        neighborhood_radius=task.parameters.bad_pixel_neighborhood_radius,
     )
 
     for row, column in ((5, 5), (5, 6)):