pycontrails 0.52.2__tar.gz → 0.52.3__tar.gz

{pycontrails-0.52.2 → pycontrails-0.52.3}/.github/workflows/release.yaml

@@ -40,7 +40,7 @@ jobs:
 
       # https://cibuildwheel.readthedocs.io/en/stable/options/#testing
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.19
+        uses: pypa/cibuildwheel@v2.20
         env:
           CIBW_BUILD: cp39-* cp310-* cp311-* cp312-*
           CIBW_SKIP: '*-win32 *-manylinux_i686 *-musllinux*'

{pycontrails-0.52.2 → pycontrails-0.52.3}/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## v0.52.3
+
+### Features
+
+- Add experimental `preprocess_lowmem` parameter to `Cocip`. When set to `True`, `Cocip` will attempt to reduce memory consumption during flight preprocessing and initial formation/persistence calculations by using an alternate implementation of `MetDataArray.interpolate` (see below).
+- Add `lowmem` keyword-only argument to `MetDataArray.interpolate`. When `True`, attempt to reduce memory consumption by using an alternative interpolation strategy that loads at most two time steps of meteorology data into memory at a time.
+
+### Fixes
+
+- Defer import of `matplotlib` in `models.cocip.output_formats`.
+- Fix bug in `PycontrailsRegularGridInterpolator` that caused errors when dispatching to 1-d linear interpolation from in `rgi_cython.pyx`.
+
+### Internals
+
+- Implement low-memory paths in `Cocip.eval` and `MetDataArray.interpolate`.
+
 ## v0.52.2
 
 ### Breaking changes

{pycontrails-0.52.2/pycontrails.egg-info → pycontrails-0.52.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pycontrails
-Version: 0.52.2
+Version: 0.52.3
 Summary: Python library for modeling aviation climate impacts
 Author-email: Breakthrough Energy <py@contrails.org>
 License: Apache-2.0

{pycontrails-0.52.2 → pycontrails-0.52.3}/docs/api.rst

@@ -134,6 +134,7 @@ CoCiP
 
     models.cocip.Cocip
     models.cocip.CocipParams
+    models.cocip.CocipFlightParams
     models.cocip.contrail_properties
     models.cocip.radiative_forcing
     models.cocip.wake_vortex

{pycontrails-0.52.2 → pycontrails-0.52.3}/pycontrails/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.52.2'
-__version_tuple__ = version_tuple = (0, 52, 2)
+__version__ = version = '0.52.3'
+__version_tuple__ = version_tuple = (0, 52, 3)

{pycontrails-0.52.2 → pycontrails-0.52.3}/pycontrails/core/interpolation.py

@@ -215,7 +215,8 @@ class PycontrailsRegularGridInterpolator(scipy.interpolate.RegularGridInterpolat
 
         if ndim == 1:
             # np.interp could be better ... although that may also promote the dtype
-            return rgi_cython.evaluate_linear_1d(values, indices, norm_distances, out)
+            # 1-d view is required for evaluate_linear_1d
+            return rgi_cython.evaluate_linear_1d(values, indices[0, :], norm_distances[0, :], out)
 
         msg = f"Invalid number of dimensions: {ndim}"
         raise ValueError(msg)

{pycontrails-0.52.2 → pycontrails-0.52.3}/pycontrails/core/met.py

@@ -9,7 +9,15 @@ import pathlib
 import typing
 import warnings
 from abc import ABC, abstractmethod
-from collections.abc import Hashable, Iterable, Iterator, Mapping, MutableMapping, Sequence
+from collections.abc import (
+    Generator,
+    Hashable,
+    Iterable,
+    Iterator,
+    Mapping,
+    MutableMapping,
+    Sequence,
+)
 from contextlib import ExitStack
 from datetime import datetime
 from typing import (
@@ -1502,6 +1510,7 @@ class MetDataArray(MetBase):
         bounds_error: bool = ...,
         fill_value: float | np.float64 | None = ...,
         localize: bool = ...,
+        lowmem: bool = ...,
         indices: interpolation.RGIArtifacts | None = ...,
         return_indices: Literal[False] = ...,
     ) -> npt.NDArray[np.float64]: ...
@@ -1518,6 +1527,7 @@ class MetDataArray(MetBase):
         bounds_error: bool = ...,
         fill_value: float | np.float64 | None = ...,
         localize: bool = ...,
+        lowmem: bool = ...,
         indices: interpolation.RGIArtifacts | None = ...,
         return_indices: Literal[True],
     ) -> tuple[npt.NDArray[np.float64], interpolation.RGIArtifacts]: ...
@@ -1533,6 +1543,7 @@ class MetDataArray(MetBase):
         bounds_error: bool = False,
         fill_value: float | np.float64 | None = np.nan,
         localize: bool = False,
+        lowmem: bool = False,
         indices: interpolation.RGIArtifacts | None = None,
         return_indices: bool = False,
     ) -> npt.NDArray[np.float64] | tuple[npt.NDArray[np.float64], interpolation.RGIArtifacts]:
@@ -1540,7 +1551,9 @@ class MetDataArray(MetBase):
 
         Zero dimensional coordinates are reshaped to 1D arrays.
 
-        Method automatically loads underlying :attr:`data` into memory.
+        If ``lowmem == False``, method automatically loads underlying :attr:`data` into
+        memory. Otherwise, method iterates through smaller subsets of :attr:`data` and releases
+        subsets from memory once interpolation against each subset is finished.
 
         If ``method == "nearest"``, the out array will have the same ``dtype`` as
         the underlying :attr:`data`.
@@ -1586,10 +1599,18 @@ class MetDataArray(MetBase):
         localize: bool, optional
             Experimental. If True, downselect gridded data to smallest bounding box containing
             all points.  By default False.
+        lowmem: bool, optional
+            Experimental. If True, iterate through points binned by the time coordinate of the
+            grided data, and downselect gridded data to the smallest bounding box containing
+            each binned set of point *before loading into memory*. This can significantly reduce
+            memory consumption with large numbers of points at the cost of increased runtime.
+            By default False.
         indices: tuple | None, optional
             Experimental. See :func:`interpolation.interp`. None by default.
         return_indices: bool, optional
             Experimental. See :func:`interpolation.interp`. False by default.
+            Note that values returned differ when ``lowmem=True`` and ``lowmem=False``,
+            so output should only be re-used in calls with the same ``lowmem`` value.
 
         Returns
         -------
@@ -1632,10 +1653,29 @@ class MetDataArray(MetBase):
         >>> level = np.linspace(200, 300, 10)
         >>> time = pd.date_range("2022-03-01T14", periods=10, freq="5min")
         >>> mda.interpolate(longitude, latitude, level, time)
+        array([220.44347694, 223.08900738, 225.74338924, 228.41642088,
+               231.10858599, 233.54857391, 235.71504913, 237.86478872,
+               239.99274623, 242.10792167])
+
+        >>> # Can easily switch to alternative low-memory implementation
+        >>> mda.interpolate(longitude, latitude, level, time, lowmem=True)
         array([220.44347694, 223.08900738, 225.74338924, 228.41642088,
                231.10858599, 233.54857391, 235.71504913, 237.86478872,
                239.99274623, 242.10792167])
         """
+        if lowmem:
+            return self._interp_lowmem(
+                longitude,
+                latitude,
+                level,
+                time,
+                method=method,
+                bounds_error=bounds_error,
+                fill_value=fill_value,
+                indices=indices,
+                return_indices=return_indices,
+            )
+
         # Load if necessary
         if not self.in_memory:
             self._check_memory("Interpolation over")
@@ -1660,6 +1700,100 @@ class MetDataArray(MetBase):
             return_indices=return_indices,
         )
 
+    def _interp_lowmem(
+        self,
+        longitude: float | npt.NDArray[np.float64],
+        latitude: float | npt.NDArray[np.float64],
+        level: float | npt.NDArray[np.float64],
+        time: np.datetime64 | npt.NDArray[np.datetime64],
+        *,
+        method: str = "linear",
+        bounds_error: bool = False,
+        fill_value: float | np.float64 | None = np.nan,
+        minimize_memory: bool = False,
+        indices: interpolation.RGIArtifacts | None = None,
+        return_indices: bool = False,
+    ) -> npt.NDArray[np.float64] | tuple[npt.NDArray[np.float64], interpolation.RGIArtifacts]:
+        """Interpolate values against underlying DataArray.
+
+        This method is used by :meth:`interpolate` when ``lowmem=True``.
+        Parameters and return types are identical to :meth:`interpolate`, except
+        that the ``localize`` keyword argument is omitted.
+        """
+        # Convert all inputs to 1d arrays
+        # Not validating against ndim >= 2
+        longitude, latitude, level, time = np.atleast_1d(longitude, latitude, level, time)
+
+        if bounds_error:
+            _lowmem_boundscheck(time, self.data)
+
+        # Create buffers for holding interpolation output
+        # Use np.full rather than np.empty so points not covered
+        # by masks are filled with correct out-of-bounds values.
+        out = np.full(longitude.shape, fill_value, dtype=self.data.dtype)
+        if return_indices:
+            rgi_artifacts = interpolation.RGIArtifacts(
+                xi_indices=np.full((4, longitude.size), -1, dtype=np.int64),
+                norm_distances=np.full((4, longitude.size), np.nan, dtype=np.float64),
+                out_of_bounds=np.full((longitude.size,), True, dtype=np.bool_),
+            )
+
+        # Iterate over portions of points between adjacent time steps in gridded data
+        for mask in _lowmem_masks(time, self.data["time"].values):
+            if mask is None or not np.any(mask):
+                continue
+
+            lon_sl = longitude[mask]
+            lat_sl = latitude[mask]
+            lev_sl = level[mask]
+            t_sl = time[mask]
+            if indices is not None:
+                indices_sl = interpolation.RGIArtifacts(
+                    xi_indices=indices.xi_indices[:, mask],
+                    norm_distances=indices.norm_distances[:, mask],
+                    out_of_bounds=indices.out_of_bounds[mask],
+                )
+            else:
+                indices_sl = None
+
+            coords = {"longitude": lon_sl, "latitude": lat_sl, "level": lev_sl, "time": t_sl}
+            if any(np.all(np.isnan(coord)) for coord in coords.values()):
+                continue
+            da = interpolation._localize(self.data, coords)
+            if not da._in_memory:
+                logger.debug(
+                    "Loading %s MB subset of %s into memory.",
+                    round(da.nbytes / 1_000_000, 2),
+                    da.name,
+                )
+                da.load()
+
+            tmp = interpolation.interp(
+                longitude=lon_sl,
+                latitude=lat_sl,
+                level=lev_sl,
+                time=t_sl,
+                da=da,
+                method=method,
+                bounds_error=bounds_error,
+                fill_value=fill_value,
+                localize=False,  # would be no-op; da is localized already
+                indices=indices_sl,
+                return_indices=return_indices,
+            )
+
+            if return_indices:
+                out[mask], rgi_sl = tmp
+                rgi_artifacts.xi_indices[:, mask] = rgi_sl.xi_indices
+                rgi_artifacts.norm_distances[:, mask] = rgi_sl.norm_distances
+                rgi_artifacts.out_of_bounds[mask] = rgi_sl.out_of_bounds
+            else:
+                out[mask] = tmp
+
+        if return_indices:
+            return out, rgi_artifacts
+        return out
+
     def _check_memory(self, msg_start: str) -> None:
         """Check the memory usage of the underlying data.
 
@@ -2656,3 +2790,46 @@ def _add_vertical_coords(data: XArrayType) -> XArrayType:
         data.coords["altitude"] = data.coords["altitude"].astype(dtype, copy=False)
 
     return data
+
+
+def _lowmem_boundscheck(time: npt.NDArray[np.datetime64], da: xr.DataArray) -> None:
+    """Extra bounds check required with low-memory interpolation strategy.
+
+    Because the main loop in `_interp_lowmem` processes points between time steps
+    in gridded data, it will never encounter points that are out-of-bounds in time
+    and may fail to produce requested out-of-bounds errors.
+    """
+    da_time = da["time"].to_numpy()
+    if not np.all((time >= da_time.min()) & (time <= da_time.max())):
+        axis = da.get_axis_num("time")
+        msg = f"One of the requested xi is out of bounds in dimension {axis}"
+        raise ValueError(msg)
+
+
+def _lowmem_masks(
+    time: npt.NDArray[np.datetime64], t_met: npt.NDArray[np.datetime64]
+) -> Generator[npt.NDArray[np.bool_], None, None]:
+    """Generate sequence of masks for low-memory interpolation."""
+    t_met_max = t_met.max()
+    t_met_min = t_met.min()
+    inbounds = (time >= t_met_min) & (time <= t_met_max)
+    if not np.any(inbounds):
+        return
+
+    earliest = np.nanmin(time)
+    istart = 0 if earliest < t_met_min else np.flatnonzero(t_met <= earliest).max()
+    latest = np.nanmax(time)
+    iend = t_met.size - 1 if latest > t_met_max else np.flatnonzero(t_met >= latest).min()
+    if istart == iend:
+        yield inbounds
+        return
+
+    # Sequence of masks covers elements in time in the interval [t_met[istart], t_met[iend]].
+    # The first iteration masks elements in the interval [t_met[istart], t_met[istart+1]]
+    # (inclusive of both endpoints).
+    # Subsequent iterations mask elements in the interval (t_met[i], t_met[i+1]]
+    # (inclusive of right endpoint only).
+    for i in range(istart, iend):
+        mask = ((time >= t_met[i]) if i == istart else (time > t_met[i])) & (time <= t_met[i + 1])
+        if np.any(mask):
+            yield mask