dkist-processing-dlnirsp 0.34.2__py3-none-any.whl → 0.34.3__py3-none-any.whl

dkist_processing_dlnirsp/tasks/movie.py

@@ -262,10 +262,9 @@ class MakeDlnirspMovie(DlnirspTaskBase):
         logger.info("Parsing headers")
         wcs_list = []
         for f in file_list:
-            hdul = fits.open(f)
-            header = self.add_L1_header_values(hdul[0].header)
+            raw_header = fits.getheader(f)
+            header = self.add_L1_header_values(raw_header)
             wcs_list.append(WCS(header).dropaxis(2))
-            del hdul
 
         logger.info("Computing reference WCS")
         reference_wcs, shape_out = find_optimal_celestial_wcs(
@@ -829,18 +828,33 @@ class MakeDlnirspMovie(DlnirspTaskBase):
         date_beg_list = []
         data_list = [[], []]
         for fo in file_paths:
-            hdul = fits.open(fo)
+            # `memmap = True` saves us a bit in terms of peak memory in this function.
+            # See below for how we make sure the open file descriptors don't linger
+            hdul = fits.open(fo, memmap=True)
             header = self.add_L1_header_values(hdul[0].header)
             wcs_list.append(WCS(header).dropaxis(2))
             date_beg_list.append(header["DATE-BEG"])
 
             for i, s in enumerate([self.core_wave_idx, self.cont_wave_idx]):
-                data = hdul[0].data[s, :, :]
+                # By assigning `hdul[0].data` to a variable explicitly we can ensure its FD will be cleaned up
+                # once we're done with it.
+                full_data = hdul[0].data
+                # The `float32` casting has two functions: First, it acts like `.copy()` in that it decouples `data`
+                # from the `full_data` variable (i.e., it's not just a view). This lets us clean up the FD used by
+                # `full_data` while keeping the much smaller `data` around. Secondly, the original data type is float64
+                # and this reduces the absolute size of `data` in memory. This task is just for a visualization, so
+                # those extra 32 bits really don't matter.
+                data = full_data[s, :, :].astype(np.float32)
                 for vs in self.parameters.movie_vertical_nan_slices:
                     data[vs, :] = np.nan
                 data_list[i].append(data)
 
-            del hdul
+            # These two lines aren't strictly necessary because these variables will be "cleaned up" when they are
+            # reused in the next loop iteration, but keeping them around makes it clear what our intentions are;
+            # these are the way to guarantee that the `hdul` opened with `memmap=True` closes both of its open file
+            # descriptors.
+            hdul.close()
+            del full_data
 
         return data_list, wcs_list, date_beg_list
 
@@ -892,7 +906,7 @@ class MakeDlnirspMovie(DlnirspTaskBase):
 
     def get_stacked_spectra(self, file_path: Path) -> np.ndarray:
         """Read a file for a single mosaic step and flatten the 3D cube into a stacked, 2D spectral array."""
-        data = fits.getdata(file_path)
+        data = fits.getdata(file_path, memmap=False)
         stack = data.reshape(self.num_wave, self.num_spec).T
         return stack
 

dkist_processing_dlnirsp/tests/test_movie.py

@@ -1,7 +1,9 @@
-import shutil
+import os
+from pathlib import Path
 
 import numpy as np
 import pytest
+from astropy.wcs import WCS
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.codecs.json import json_encoder
 
@@ -13,13 +15,51 @@ from dkist_processing_dlnirsp.tests.conftest import DlnirspTestingParameters
 from dkist_processing_dlnirsp.tests.conftest import write_calibrated_frames_to_task
 
 
+def check_open_file_descriptors() -> int | None:
+    """
+    Look for the number of open file descriptors by examining /proc/self/fd.
+
+    This concept doesn't apply to Windows machines, so we will catch that error gracefully. The result is that Windows
+    won't test this condition, but that's OK because it doesn't apply to them anyway.
+    """
+    try:
+        return len(os.listdir("/proc/self/fd"))
+    except FileNotFoundError:
+        return None
+
+
+class LeakFindingMovieTask(MakeDlnirspMovie):
+    """Same as `MakeDlnirspMovie`, but keeps track of file descriptors before and after each call to potentially leaky functions."""
+
+    def pre_run(self) -> None:
+        super().pre_run()
+        self._test_pre_mosaic_num_fd = []
+        self._test_post_mosaic_num_fd = []
+        self._test_pre_stack_num_fd = []
+        self._test_post_stack_num_fd = []
+
+    def read_mosaic_frames(
+        self, mosaic_num: int, stokes: str
+    ) -> tuple[list[list[np.ndarray]], list[WCS], list[str]]:
+        self._test_pre_mosaic_num_fd.append(check_open_file_descriptors())
+        result = super().read_mosaic_frames(mosaic_num, stokes)
+        self._test_post_mosaic_num_fd.append(check_open_file_descriptors())
+        return result
+
+    def get_stacked_spectra(self, file_path: Path) -> np.ndarray:
+        self._test_pre_stack_num_fd.append(check_open_file_descriptors())
+        result = super().get_stacked_spectra(file_path)
+        self._test_post_stack_num_fd.append(check_open_file_descriptors())
+        return result
+
+
 @pytest.fixture
 def movie_task(recipe_run_id, tmp_path, link_constants_db, assign_input_dataset_doc_to_task):
     link_constants_db(
         recipe_run_id=recipe_run_id,
         constants_obj=DlnirspTestingConstants(),
     )
-    with MakeDlnirspMovie(
+    with LeakFindingMovieTask(
         recipe_run_id=recipe_run_id,
         workflow_name="workflow_name",
         workflow_version="workflow_version",
@@ -139,3 +179,14 @@ def test_movie_task(
     movie_file_list = list(task.read(tags=[DlnirspTag.output(), DlnirspTag.movie()]))
 
     assert len(movie_file_list) == 1
+
+    # Check that there are no file descriptor leaks in the functions that could be leaky
+    #
+    # Using `xdist` can cause some timing issues with these tests.
+    # The "true" assert is `after == before`, but tests have shown that sometimes `after` can be lower than `before`,
+    # presumably because some fixture cleanup or something happens in a race with the checking of open FD that occurs
+    # in `LeakFindingMovieTask`. To deal with this we just test `<=`, which still captures the intent with no downside.
+    for before, after in zip(task._test_pre_mosaic_num_fd, task._test_post_mosaic_num_fd):
+        assert after <= before
+    for before, after in zip(task._test_pre_stack_num_fd, task._test_post_stack_num_fd):
+        assert after <= before

{dkist_processing_dlnirsp-0.34.2.dist-info → dkist_processing_dlnirsp-0.34.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-dlnirsp
-Version: 0.34.2
+Version: 0.34.3
 Summary: Science processing code for the DLNIRSP instrument on DKIST
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.13
 Description-Content-Type: text/x-rst
-Requires-Dist: dkist-processing-common==12.2.0
+Requires-Dist: dkist-processing-common==12.3.0
 Requires-Dist: dkist-processing-math==2.2.1
 Requires-Dist: dkist-processing-pac==3.1.1
 Requires-Dist: dkist-header-validator==5.3.0
@@ -93,13 +93,13 @@ Requires-Dist: annotated-types==0.7.0; extra == "frozen"
 Requires-Dist: anyio==4.12.1; extra == "frozen"
 Requires-Dist: apache-airflow==3.1.6; extra == "frozen"
 Requires-Dist: apache-airflow-core==3.1.6; extra == "frozen"
-Requires-Dist: apache-airflow-providers-celery==3.15.1; extra == "frozen"
-Requires-Dist: apache-airflow-providers-common-compat==1.12.0; extra == "frozen"
+Requires-Dist: apache-airflow-providers-celery==3.15.2; extra == "frozen"
+Requires-Dist: apache-airflow-providers-common-compat==1.13.0; extra == "frozen"
 Requires-Dist: apache-airflow-providers-common-io==1.7.1; extra == "frozen"
-Requires-Dist: apache-airflow-providers-common-sql==1.30.3; extra == "frozen"
-Requires-Dist: apache-airflow-providers-postgres==6.5.2; extra == "frozen"
+Requires-Dist: apache-airflow-providers-common-sql==1.30.4; extra == "frozen"
+Requires-Dist: apache-airflow-providers-postgres==6.5.3; extra == "frozen"
 Requires-Dist: apache-airflow-providers-smtp==2.4.2; extra == "frozen"
-Requires-Dist: apache-airflow-providers-standard==1.10.3; extra == "frozen"
+Requires-Dist: apache-airflow-providers-standard==1.11.0; extra == "frozen"
 Requires-Dist: apache-airflow-task-sdk==1.1.6; extra == "frozen"
 Requires-Dist: argcomplete==3.6.3; extra == "frozen"
 Requires-Dist: asdf==3.5.0; extra == "frozen"
@@ -108,14 +108,14 @@ Requires-Dist: asdf_transform_schemas==0.6.0; extra == "frozen"
 Requires-Dist: asgiref==3.11.0; extra == "frozen"
 Requires-Dist: asteval==1.0.8; extra == "frozen"
 Requires-Dist: astropy==7.0.2; extra == "frozen"
-Requires-Dist: astropy-iers-data==0.2026.1.26.0.43.56; extra == "frozen"
+Requires-Dist: astropy-iers-data==0.2026.2.2.0.48.1; extra == "frozen"
 Requires-Dist: astropy_healpix==1.1.3; extra == "frozen"
 Requires-Dist: asyncpg==0.31.0; extra == "frozen"
 Requires-Dist: attrs==25.4.0; extra == "frozen"
-Requires-Dist: babel==2.17.0; extra == "frozen"
+Requires-Dist: babel==2.18.0; extra == "frozen"
 Requires-Dist: billiard==4.2.4; extra == "frozen"
-Requires-Dist: boto3==1.42.38; extra == "frozen"
-Requires-Dist: botocore==1.42.38; extra == "frozen"
+Requires-Dist: boto3==1.42.39; extra == "frozen"
+Requires-Dist: botocore==1.42.39; extra == "frozen"
 Requires-Dist: cadwyn==5.4.6; extra == "frozen"
 Requires-Dist: celery==5.6.2; extra == "frozen"
 Requires-Dist: certifi==2026.1.4; extra == "frozen"
@@ -133,14 +133,14 @@ Requires-Dist: croniter==6.0.0; extra == "frozen"
 Requires-Dist: cryptography==46.0.4; extra == "frozen"
 Requires-Dist: cycler==0.12.1; extra == "frozen"
 Requires-Dist: dacite==1.9.2; extra == "frozen"
-Requires-Dist: dask==2026.1.1; extra == "frozen"
+Requires-Dist: dask==2026.1.2; extra == "frozen"
 Requires-Dist: dask-image==2025.11.0; extra == "frozen"
 Requires-Dist: decorator==5.2.1; extra == "frozen"
 Requires-Dist: dill==0.4.1; extra == "frozen"
 Requires-Dist: dkist-header-validator==5.3.0; extra == "frozen"
-Requires-Dist: dkist-processing-common==12.2.0; extra == "frozen"
+Requires-Dist: dkist-processing-common==12.3.0; extra == "frozen"
 Requires-Dist: dkist-processing-core==7.0.1; extra == "frozen"
-Requires-Dist: dkist-processing-dlnirsp==0.34.2; extra == "frozen"
+Requires-Dist: dkist-processing-dlnirsp==0.34.3; extra == "frozen"
 Requires-Dist: dkist-processing-math==2.2.1; extra == "frozen"
 Requires-Dist: dkist-processing-pac==3.1.1; extra == "frozen"
 Requires-Dist: dkist-service-configuration==4.2.0; extra == "frozen"
@@ -235,7 +235,7 @@ Requires-Dist: pathspec==1.0.4; extra == "frozen"
 Requires-Dist: pendulum==3.2.0; extra == "frozen"
 Requires-Dist: pika==1.3.2; extra == "frozen"
 Requires-Dist: pillow==11.3.0; extra == "frozen"
-Requires-Dist: pip==25.3; extra == "frozen"
+Requires-Dist: pip==26.0; extra == "frozen"
 Requires-Dist: platformdirs==4.5.1; extra == "frozen"
 Requires-Dist: pluggy==1.6.0; extra == "frozen"
 Requires-Dist: pooch==1.9.0; extra == "frozen"
@@ -264,9 +264,9 @@ Requires-Dist: redis==6.4.0; extra == "frozen"
 Requires-Dist: referencing==0.37.0; extra == "frozen"
 Requires-Dist: reproject==0.19.0; extra == "frozen"
 Requires-Dist: requests==2.32.5; extra == "frozen"
-Requires-Dist: rich==14.3.1; extra == "frozen"
+Requires-Dist: rich==14.3.2; extra == "frozen"
 Requires-Dist: rich-argparse==1.7.2; extra == "frozen"
-Requires-Dist: rich-toolkit==0.17.2; extra == "frozen"
+Requires-Dist: rich-toolkit==0.18.1; extra == "frozen"
 Requires-Dist: rpds-py==0.30.0; extra == "frozen"
 Requires-Dist: s3transfer==0.16.0; extra == "frozen"
 Requires-Dist: scikit-image==0.26.0; extra == "frozen"
@@ -292,7 +292,7 @@ Requires-Dist: text-unidecode==1.3; extra == "frozen"
 Requires-Dist: tifffile==2026.1.28; extra == "frozen"
 Requires-Dist: toolz==1.1.0; extra == "frozen"
 Requires-Dist: tornado==6.5.4; extra == "frozen"
-Requires-Dist: tqdm==4.67.1; extra == "frozen"
+Requires-Dist: tqdm==4.67.2; extra == "frozen"
 Requires-Dist: traitlets==5.14.3; extra == "frozen"
 Requires-Dist: typer==0.21.1; extra == "frozen"
 Requires-Dist: typing-inspection==0.4.2; extra == "frozen"
@@ -309,7 +309,7 @@ Requires-Dist: uvloop==0.22.1; extra == "frozen"
 Requires-Dist: vine==5.1.0; extra == "frozen"
 Requires-Dist: voluptuous==0.16.0; extra == "frozen"
 Requires-Dist: watchfiles==1.1.1; extra == "frozen"
-Requires-Dist: wcwidth==0.5.2; extra == "frozen"
+Requires-Dist: wcwidth==0.5.3; extra == "frozen"
 Requires-Dist: websockets==16.0; extra == "frozen"
 Requires-Dist: wirerope==1.0.0; extra == "frozen"
 Requires-Dist: wrapt==1.17.3; extra == "frozen"

{dkist_processing_dlnirsp-0.34.2.dist-info → dkist_processing_dlnirsp-0.34.3.dist-info}/RECORD

@@ -26,7 +26,7 @@ dkist_processing_dlnirsp/tasks/instrument_polarization.py,sha256=G79qXg1080zsd8Z
 dkist_processing_dlnirsp/tasks/l1_output_data.py,sha256=Psta0476jih-cfmpdnzmquys030sW4f4VX6ksAb-g1o,465
 dkist_processing_dlnirsp/tasks/lamp.py,sha256=twoMHMnE2jm2OYxkeHFcjqcyQYVtEjSU2_z2mmn3Ybs,5390
 dkist_processing_dlnirsp/tasks/linearity_correction.py,sha256=z3MpWHp0lZYB2zrfHfH7HlcKoro6TPxwEN8rpqVv08c,22716
-dkist_processing_dlnirsp/tasks/movie.py,sha256=IaZCRg2KIETEXDyyOe3FLoY5cd7eJOhMtjkiSYjElEU,44291
+dkist_processing_dlnirsp/tasks/movie.py,sha256=sRR_1GsKlSUiHerHkmNTEBC_tsrj7k8_EuZzgfvZHO0,45614
 dkist_processing_dlnirsp/tasks/parse.py,sha256=shhepxnN2s3u9EFfMOAIaIN-be5Wtsaw0H6Trh5kOOQ,11359
 dkist_processing_dlnirsp/tasks/quality_metrics.py,sha256=1ScLAinv3WG-0M7u_26gnogbJ_D84K9CP7eRvEuwb9E,7346
 dkist_processing_dlnirsp/tasks/science.py,sha256=sNI4AYO10qe3JL62KWXR87MnfX9m85Jo4_PkQ0qd5TQ,35737
@@ -51,7 +51,7 @@ dkist_processing_dlnirsp/tests/test_ifu_drift.py,sha256=K6ushbVQVE6AATjiP3hEgF6o
 dkist_processing_dlnirsp/tests/test_instrument_polarization_calibration.py,sha256=pPXjfzi3EcaawNqvXsz3OcCF1XITEJeCzQ8-iOjlm7U,23430
 dkist_processing_dlnirsp/tests/test_lamp.py,sha256=umy4ydYR5M3dqXTQP14qbamOleutyD1jSj3325aitl0,3940
 dkist_processing_dlnirsp/tests/test_linearity_correction.py,sha256=JkPNtsRad2JQQ2QiJjQar9tJuCi9-ksOdEYHW_-kl58,31793
-dkist_processing_dlnirsp/tests/test_movie.py,sha256=j1ocVuy7PZCJrL-MrTczI-mFEsnECyCQ6P8uiK3mokw,4284
+dkist_processing_dlnirsp/tests/test_movie.py,sha256=fDfXi-nGsKSl0oY2OQEtT4tMHozRFIevQ5pqrgyk7bI,6627
 dkist_processing_dlnirsp/tests/test_parameters.py,sha256=bL30vGwFMwY2vP6-mQ-KPsr7I213PaXP7S1wKVPqq8g,10705
 dkist_processing_dlnirsp/tests/test_parse.py,sha256=prx8K0vFGkW-YZm0YB13FBobbiMf0u1k8TuCyNF4_qE,38418
 dkist_processing_dlnirsp/tests/test_quality.py,sha256=OBWva_E6T5RfDuo73LiTdTr7uh2FO04VpBAsSOLfjVQ,6122
@@ -91,8 +91,8 @@ docs/science_calibration.rst,sha256=JUMeS7KyIsL9cUuy-IdV2tQ-wQ2lBx6j8u_51ThJzVg,
 docs/scientific_changelog.rst,sha256=01AWBSHg8zElnodCgAq-hMxhk9CkX5rtEENx4iz0sjI,300
 docs/wavelength_calibration.rst,sha256=QGUd5Xmv5f7WQ1EKvw2CcgiMJQULjzdCW1hSQ8h8jj0,4123
 licenses/LICENSE.rst,sha256=piZaQplkzOMmH1NXg6QIdo9wwo9pPCoHkvm2-DmH76E,1462
-dkist_processing_dlnirsp-0.34.2.dist-info/METADATA,sha256=w-tDHGKezAPomaY4ReckGYDsS1wmgEFwOYMlw4N2Jtk,30435
-dkist_processing_dlnirsp-0.34.2.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-dkist_processing_dlnirsp-0.34.2.dist-info/entry_points.txt,sha256=p4-7cpIfxmQGFUDIP5n5noE5KADHN2-JvV03e0gOx9s,140
-dkist_processing_dlnirsp-0.34.2.dist-info/top_level.txt,sha256=4WmLV9LQM78KTFnkHmtaOJvVHAPpz0m9ruzDS-B_cUo,49
-dkist_processing_dlnirsp-0.34.2.dist-info/RECORD,,
+dkist_processing_dlnirsp-0.34.3.dist-info/METADATA,sha256=VRViQgISghksCNybqEEYFjN85mLICPGiHRleB2S9MAg,30433
+dkist_processing_dlnirsp-0.34.3.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+dkist_processing_dlnirsp-0.34.3.dist-info/entry_points.txt,sha256=p4-7cpIfxmQGFUDIP5n5noE5KADHN2-JvV03e0gOx9s,140
+dkist_processing_dlnirsp-0.34.3.dist-info/top_level.txt,sha256=4WmLV9LQM78KTFnkHmtaOJvVHAPpz0m9ruzDS-B_cUo,49
+dkist_processing_dlnirsp-0.34.3.dist-info/RECORD,,