pycontrails 0.53.1__cp310-cp310-macosx_11_0_arm64.whl → 0.54.0__cp310-cp310-macosx_11_0_arm64.whl

pycontrails/datalib/ecmwf/era5.py

@@ -41,6 +41,7 @@ class ERA5(ECMWFAPI):
     or as ``CDSAPI_URL`` and ``CDSAPI_KEY`` environment variables.
 
         export CDSAPI_URL=...
+
         export CDSAPI_KEY=...
 
     Credentials can also be provided directly ``url`` and ``key`` keyword args.

pycontrails/datalib/ecmwf/era5_model_level.py

@@ -3,7 +3,7 @@
 This module supports
 
 - Retrieving model-level ERA5 data by submitting MARS requests through the Copernicus CDS.
-- Processing retrieved GRIB files to produce netCDF files on target pressure levels.
+- Processing retrieved model-level files to produce netCDF files on target pressure levels.
 - Local caching of processed netCDF files.
 - Opening processed and cached files as a :class:`pycontrails.MetDataset` object.
 
@@ -15,19 +15,17 @@ and has lower latency than this module, which retrieves data from the
 `Copernicus Climate Data Store <https://cds.climate.copernicus.eu/#!/home>`_.
 This module must be used to retrieve model-level data from ERA5 ensemble members
 or for more recent dates.
-
-This module requires the following additional dependency:
-
-- `metview (binaries and python bindings) <https://metview.readthedocs.io/en/latest/python.html>`_
 """
 
 from __future__ import annotations
 
 import collections
+import concurrent.futures
 import contextlib
 import hashlib
 import logging
 import os
+import threading
 import warnings
 from datetime import datetime
 from typing import Any
@@ -43,8 +41,8 @@ import pycontrails
 from pycontrails.core import cache
 from pycontrails.core.met import MetDataset, MetVariable
 from pycontrails.datalib._met_utils import metsource
+from pycontrails.datalib.ecmwf import model_levels as mlmod
 from pycontrails.datalib.ecmwf.common import ECMWFAPI, CDSCredentialsNotFound
-from pycontrails.datalib.ecmwf.model_levels import pressure_levels_at_model_levels
 from pycontrails.datalib.ecmwf.variables import MODEL_LEVEL_VARIABLES
 from pycontrails.utils import dependencies, temp
 
@@ -65,6 +63,7 @@ class ERA5ModelLevel(ECMWFAPI):
     or as ``CDSAPI_URL`` and ``CDSAPI_KEY`` environment variables.
 
         export CDSAPI_URL=...
+
         export CDSAPI_KEY=...
 
     Credentials can also be provided directly ``url`` and ``key`` keyword args.
@@ -79,7 +78,7 @@ class ERA5ModelLevel(ECMWFAPI):
         Input must be datetime-like or tuple of datetime-like
         (:py:class:`datetime.datetime`, :class:`pandas.Timestamp`, :class:`numpy.datetime64`)
         specifying the (start, end) of the date range, inclusive.
-        GRIB files will be downloaded from CDS in chunks no larger than 1 month
+        NetCDF files will be downloaded from CDS in chunks no larger than 1 month
         for the nominal reanalysis and no larger than 1 day for ensemble members.
         This ensures that exactly one request is submitted per file on tape accessed.
         If None, ``paths`` must be defined and all time coordinates will be loaded from files.
@@ -100,7 +99,7 @@ class ERA5ModelLevel(ECMWFAPI):
     grid : float, optional
         Specify latitude/longitude grid spacing in data.
         By default, this is set to 0.25 for reanalysis products and 0.5 for ensemble products.
-    levels : list[int], optional
+    model_levels : list[int], optional
         Specify ECMWF model levels to include in MARS requests.
         By default, this is set to include all model levels.
     ensemble_members : list[int], optional
@@ -111,8 +110,8 @@ class ERA5ModelLevel(ECMWFAPI):
         Cache data store for staging processed netCDF files.
         Defaults to :class:`pycontrails.core.cache.DiskCacheStore`.
         If None, cache is turned off.
-    cache_grib: bool, optional
-        If True, cache downloaded GRIB files rather than storing them in a temporary file.
+    cache_download: bool, optional
+        If True, cache downloaded model-level files rather than storing them in a temporary file.
         By default, False.
     url : str | None
         Override the default `cdsapi <https://github.com/ecmwf/cdsapi>`_ url.
@@ -133,21 +132,20 @@ class ERA5ModelLevel(ECMWFAPI):
         self,
         time: metsource.TimeInput,
         variables: metsource.VariableInput,
+        *,
         pressure_levels: metsource.PressureLevelInput | None = None,
         timestep_freq: str | None = None,
         product_type: str = "reanalysis",
         grid: float | None = None,
-        levels: list[int] | None = None,
+        model_levels: list[int] | None = None,
         ensemble_members: list[int] | None = None,
         cachestore: cache.CacheStore = __marker,  # type: ignore[assignment]
-        n_jobs: int = 1,
-        cache_grib: bool = False,
+        cache_download: bool = False,
         url: str | None = None,
         key: str | None = None,
     ) -> None:
-
         self.cachestore = cache.DiskCacheStore() if cachestore is self.__marker else cachestore
-        self.cache_grib = cache_grib
+        self.cache_download = cache_download
 
         self.paths = None
 
@@ -163,7 +161,7 @@ class ERA5ModelLevel(ECMWFAPI):
             raise ValueError(msg)
         self.product_type = product_type
 
-        if product_type == "reanalysis" and ensemble_members:
+        if product_type != "ensemble_members" and ensemble_members:
             msg = "No ensemble members available for reanalysis product type."
             raise ValueError(msg)
         if product_type == "ensemble_members" and not ensemble_members:
@@ -184,12 +182,12 @@ class ERA5ModelLevel(ECMWFAPI):
                 warnings.warn(msg)
         self.grid = grid
 
-        if levels is None:
-            levels = list(range(1, 138))
-        if min(levels) < 1 or max(levels) > 137:
-            msg = "Retrieval levels must be between 1 and 137, inclusive."
+        if model_levels is None:
+            model_levels = list(range(1, 138))
+        elif min(model_levels) < 1 or max(model_levels) > 137:
+            msg = "Retrieval model_levels must be between 1 and 137, inclusive."
             raise ValueError(msg)
-        self.levels = levels
+        self.model_levels = model_levels
 
         datasource_timestep_freq = "1h" if product_type == "reanalysis" else "3h"
         if timestep_freq is None:
@@ -203,7 +201,7 @@ class ERA5ModelLevel(ECMWFAPI):
 
         self.timesteps = metsource.parse_timesteps(time, freq=timestep_freq)
         if pressure_levels is None:
-            pressure_levels = pressure_levels_at_model_levels(20_000.0, 50_000.0)
+            pressure_levels = mlmod.model_level_reference_pressure(20_000.0, 50_000.0)
         self.pressure_levels = metsource.parse_pressure_levels(pressure_levels)
         self.variables = metsource.parse_variables(variables, self.pressure_level_variables)
 
@@ -281,7 +279,6 @@ class ERA5ModelLevel(ECMWFAPI):
 
     @overrides
     def download_dataset(self, times: list[datetime]) -> None:
-
         # group data to request by month (nominal) or by day (ensemble)
         requests: dict[datetime, list[datetime]] = collections.defaultdict(list)
         for t in times:
@@ -293,7 +290,7 @@ class ERA5ModelLevel(ECMWFAPI):
             requests[request].append(t)
 
         # retrieve and process data for each request
-        LOG.debug(f"Retrieving ERA5 data for times {times} in {len(requests)} request(s)")
+        LOG.debug(f"Retrieving ERA5 ML data for times {times} in {len(requests)} request(s)")
         for times_in_request in requests.values():
             self._download_convert_cache_handler(times_in_request)
 
@@ -304,7 +301,6 @@ class ERA5ModelLevel(ECMWFAPI):
         xr_kwargs: dict[str, Any] | None = None,
         **kwargs: Any,
     ) -> MetDataset:
-
         if dataset:
             msg = "Parameter 'dataset' is not supported for Model-level ERA5 data"
             raise ValueError(msg)
@@ -340,42 +336,56 @@ class ERA5ModelLevel(ECMWFAPI):
             product=product,
         )
 
-    def mars_request(self, times: list[datetime]) -> dict[str, str]:
-        """Generate MARS request for specific list of times.
-
-        Parameters
-        ----------
-        times : list[datetime]
-            Times included in MARS request.
+    def _mars_request_base(self, times: list[datetime]) -> dict[str, str]:
+        unique_dates = {t.strftime("%Y-%m-%d") for t in times}
+        unique_times = {t.strftime("%H:%M:%S") for t in times}
 
-        Returns
-        -------
-        dict[str, str]:
-            MARS request for submission to Copernicus CDS.
-        """
-        unique_dates = set(t.strftime("%Y-%m-%d") for t in times)
-        unique_times = set(t.strftime("%H:%M:%S") for t in times)
-        # param 152 = log surface pressure, needed for metview level conversion
-        grib_params = set((*self.variable_ecmwfids, 152))
         common = {
             "class": "ea",
             "date": "/".join(sorted(unique_dates)),
             "expver": "1",
-            "levelist": "/".join(str(lev) for lev in sorted(self.levels)),
             "levtype": "ml",
-            "param": "/".join(str(p) for p in sorted(grib_params)),
             "time": "/".join(sorted(unique_times)),
             "type": "an",
             "grid": f"{self.grid}/{self.grid}",
+            "format": "netcdf",
         }
+
         if self.product_type == "reanalysis":
             specific = {"stream": "oper"}
         elif self.product_type == "ensemble_members":
-            specific = {"stream": "enda"}
-            if self.ensemble_members is not None:  # always defined; checked to satisfy mypy
-                specific |= {"number": "/".join(str(n) for n in self.ensemble_members)}
+            if self.ensemble_members is None:
+                msg = "No ensemble members specified for ensemble product type."
+                raise ValueError(msg)
+            specific = {"stream": "enda", "number": "/".join(str(n) for n in self.ensemble_members)}
+
         return common | specific
 
+    def _mars_request_lnsp(self, times: list[datetime]) -> dict[str, str]:
+        out = self._mars_request_base(times)
+        out["param"] = "152"  # lnsp, needed for model level -> pressure level conversion
+        out["levelist"] = "1"
+        return out
+
+    def mars_request(self, times: list[datetime]) -> dict[str, str]:
+        """Generate MARS request for specific list of times.
+
+        Parameters
+        ----------
+        times : list[datetime]
+            Times included in MARS request.
+
+        Returns
+        -------
+        dict[str, str]:
+            MARS request for submission to Copernicus CDS.
+        """
+
+        out = self._mars_request_base(times)
+        out["param"] = "/".join(str(p) for p in sorted(set(self.variable_ecmwfids)))
+        out["levelist"] = "/".join(str(lev) for lev in sorted(self.model_levels))
+        return out
+
     def _set_cds(self) -> None:
         """Set the cdsapi.Client instance."""
         try:
@@ -394,13 +404,10 @@ class ERA5ModelLevel(ECMWFAPI):
         except Exception as err:
             raise CDSCredentialsNotFound from err
 
-    def _download_convert_cache_handler(
-        self,
-        times: list[datetime],
-    ) -> None:
+    def _download_convert_cache_handler(self, times: list[datetime]) -> None:
         """Download, convert, and cache ERA5 model level data.
 
-        This function builds a MARS request and retrieves a single GRIB file.
+        This function builds a MARS request and retrieves a single NetCDF file.
         The calling function should ensure that all times will be contained
         in a single file on tape in the MARS archive.
 
@@ -408,82 +415,73 @@ class ERA5ModelLevel(ECMWFAPI):
         retrieved data will include the Cartesian product of all unique
         dates and times in the list of specified times.
 
-        After retrieval, this function processes the GRIB file
+        After retrieval, this function processes the NetCDF file
         to produce the dataset specified by class attributes.
 
         Parameters
         ----------
         times : list[datetime]
             Times to download in a single MARS request.
-
-        Notes
-        -----
-        This function depends on `metview <https://metview.readthedocs.io/en/latest/python.html>`_
-        python bindings and binaries.
-
-        The lifetime of the metview import must last until processed datasets are cached
-        to avoid premature deletion of metview temporary files.
         """
-        try:
-            import metview as mv
-        except ModuleNotFoundError as exc:
-            dependencies.raise_module_not_found_error(
-                "model_level.grib_to_dataset function",
-                package_name="metview",
-                module_not_found_error=exc,
-                extra="See https://metview.readthedocs.io/en/latest/install.html for instructions.",
-            )
-        except ImportError as exc:
-            msg = "Failed to import metview"
-            raise ImportError(msg) from exc
-
         if self.cachestore is None:
             msg = "Cachestore is required to download and cache data"
             raise ValueError(msg)
 
-        stack = contextlib.ExitStack()
-        request = self.mars_request(times)
+        ml_request = self.mars_request(times)
+        lnsp_request = self._mars_request_lnsp(times)
 
-        if not self.cache_grib:
-            target = stack.enter_context(temp.temp_file())
+        stack = contextlib.ExitStack()
+        if not self.cache_download:
+            ml_target = stack.enter_context(temp.temp_file())
+            lnsp_target = stack.enter_context(temp.temp_file())
         else:
-            request_str = ";".join(f"{p}:{request[p]}" for p in sorted(request.keys()))
-            name = hashlib.md5(request_str.encode()).hexdigest()
-            target = self.cachestore.path(f"era5ml-{name}.grib")
+            ml_target = _target_path(ml_request, self.cachestore)
+            lnsp_target = _target_path(lnsp_request, self.cachestore)
 
         with stack:
-            if not self.cache_grib or not self.cachestore.exists(target):
-                if not hasattr(self, "cds"):
-                    self._set_cds()
-                self.cds.retrieve("reanalysis-era5-complete", request, target)
-
-            # Read contents of GRIB file as metview Fieldset
-            LOG.debug("Opening GRIB file")
-            fs_ml = mv.read(target)
-
-            # reduce memory overhead by cacheing one timestep at a time
-            for time in times:
-                fs_pl = mv.Fieldset()
-                dimensions = self.ensemble_members if self.ensemble_members else [-1]
-                for ens in dimensions:
-                    date = time.strftime("%Y%m%d")
-                    t = time.strftime("%H%M")
-                    selection = dict(date=date, time=t)
-                    if ens >= 0:
-                        selection |= dict(number=str(ens))
-
-                    lnsp = fs_ml.select(shortName="lnsp", **selection)
-                    for var in self.variables:
-                        LOG.debug(
-                            f"Converting {var.short_name} at {t}"
-                            + (f" (ensemble member {ens})" if ens else "")
+            threads = []
+            for request, target in ((ml_request, ml_target), (lnsp_request, lnsp_target)):
+                if not self.cache_download or not self.cachestore.exists(target):
+                    if not hasattr(self, "cds"):
+                        self._set_cds()
+                    threads.append(
+                        threading.Thread(
+                            target=self.cds.retrieve,
+                            args=("reanalysis-era5-complete", request, target),
                         )
-                        f_ml = fs_ml.select(shortName=var.short_name, **selection)
-                        f_pl = mv.mvl_ml2hPa(lnsp, f_ml, self.pressure_levels)
-                        fs_pl = mv.merge(fs_pl, f_pl)
-
-                # Create, validate, and cache dataset
-                ds = fs_pl.to_dataset()
-                ds = ds.rename(isobaricInhPa="level").expand_dims("time")
-                ds.attrs["pycontrails_version"] = pycontrails.__version__
-                self.cache_dataset(ds)
+                    )
+
+            # Download across two threads
+            with concurrent.futures.ThreadPoolExecutor() as executor:
+                for thread in threads:
+                    executor.submit(thread.run)
+
+            LOG.debug("Opening model level data file")
+
+            ds_ml = xr.open_dataset(ml_target)
+            lnsp = xr.open_dataarray(lnsp_target)
+
+            # New CDS-Beta gives "valid_time" instead of "time"
+            if "valid_time" in ds_ml:
+                ds_ml = ds_ml.rename(valid_time="time")
+            if "valid_time" in lnsp.dims:
+                lnsp = lnsp.rename(valid_time="time")
+
+            # The legacy CDS gives "level" instead of "model_level"
+            if "level" in ds_ml.dims:
+                ds_ml = ds_ml.rename(level="model_level")
+
+            # Use a chunking scheme harmonious with self.cache_dataset, which groups by time
+            # Because ds_ml is dask-backed, nothing gets computed until cache_dataset is called
+            ds_ml = ds_ml.chunk(time=1)
+            lnsp = lnsp.chunk(time=1)
+
+            ds = mlmod.ml_to_pl(ds_ml, target_pl=self.pressure_levels, lnsp=lnsp)
+            ds.attrs["pycontrails_version"] = pycontrails.__version__
+            self.cache_dataset(ds)
+
+
+def _target_path(request: dict[str, str], cachestore: cache.CacheStore) -> str:
+    request_str = ";".join(f"{p}:{request[p]}" for p in sorted(request))
+    name = hashlib.md5(request_str.encode()).hexdigest()
+    return cachestore.path(f"era5ml-{name}-raw.nc")

pycontrails/datalib/ecmwf/hres_model_level.py

@@ -3,13 +3,9 @@
 This module supports
 
 - Retrieving model-level HRES data by submitting MARS requests through the ECMWF API.
-- Processing retrieved GRIB files to produce netCDF files on target pressure levels.
+- Processing retrieved model-level files to produce netCDF files on target pressure levels.
 - Local caching of processed netCDF files.
 - Opening processed and cached files as a :class:`pycontrails.MetDataset` object.
-
-This module requires the following additional dependency:
-
-- `metview (binaries and python bindings) <https://metview.readthedocs.io/en/latest/python.html>`_
 """
 
 from __future__ import annotations
@@ -31,8 +27,8 @@ import pycontrails
 from pycontrails.core import cache
 from pycontrails.core.met import MetDataset, MetVariable
 from pycontrails.datalib._met_utils import metsource
+from pycontrails.datalib.ecmwf import model_levels as mlmod
 from pycontrails.datalib.ecmwf.common import ECMWFAPI
-from pycontrails.datalib.ecmwf.model_levels import pressure_levels_at_model_levels
 from pycontrails.datalib.ecmwf.variables import MODEL_LEVEL_VARIABLES
 from pycontrails.utils import dependencies, temp
 from pycontrails.utils.types import DatetimeLike
@@ -76,7 +72,7 @@ class HRESModelLevel(ECMWFAPI):
         Input must be datetime-like or tuple of datetime-like
         (:py:class:`datetime.datetime`, :class:`pandas.Timestamp`, :class:`numpy.datetime64`)
         specifying the (start, end) of the date range, inclusive.
-        All times will be downloaded in a single GRIB file, which
+        All times will be downloaded in a single NetCDF file, which
         ensures that exactly one request is submitted per file on tape accessed.
         If ``forecast_time`` is unspecified, the forecast time will
         be assumed to be the nearest synoptic hour available in the operational archive (00 or 12).
@@ -105,8 +101,8 @@ class HRESModelLevel(ECMWFAPI):
         Cache data store for staging processed netCDF files.
         Defaults to :class:`pycontrails.core.cache.DiskCacheStore`.
         If None, cache is turned off.
-    cache_grib: bool, optional
-        If True, cache downloaded GRIB files rather than storing them in a temporary file.
+    cache_download: bool, optional
+        If True, cache downloaded NetCDF files rather than storing them in a temporary file.
         By default, False.
     url : str
         Override `ecmwf-api-client <https://github.com/ecmwf/ecmwf-api-client>`_ url
@@ -126,10 +122,9 @@ class HRESModelLevel(ECMWFAPI):
         timestep_freq: str | None = None,
         grid: float | None = None,
         forecast_time: DatetimeLike | None = None,
-        levels: list[int] | None = None,
-        ensemble_members: list[int] | None = None,
+        model_levels: list[int] | None = None,
         cachestore: cache.CacheStore = __marker,  # type: ignore[assignment]
-        cache_grib: bool = False,
+        cache_download: bool = False,
         url: str | None = None,
         key: str | None = None,
         email: str | None = None,
@@ -137,7 +132,7 @@ class HRESModelLevel(ECMWFAPI):
         # Parse and set each parameter to the instance
 
         self.cachestore = cache.DiskCacheStore() if cachestore is self.__marker else cachestore
-        self.cache_grib = cache_grib
+        self.cache_download = cache_download
 
         self.paths = None
 
@@ -159,12 +154,12 @@ class HRESModelLevel(ECMWFAPI):
                 warnings.warn(msg)
         self.grid = grid
 
-        if levels is None:
-            levels = list(range(1, 138))
-        if min(levels) < 1 or max(levels) > 137:
-            msg = "Retrieval levels must be between 1 and 137, inclusive."
+        if model_levels is None:
+            model_levels = list(range(1, 138))
+        elif min(model_levels) < 1 or max(model_levels) > 137:
+            msg = "Retrieval model_levels must be between 1 and 137, inclusive."
             raise ValueError(msg)
-        self.levels = levels
+        self.model_levels = model_levels
 
         forecast_hours = metsource.parse_timesteps(time, freq="1h")
         if forecast_time is None:
@@ -203,7 +198,7 @@ class HRESModelLevel(ECMWFAPI):
             raise ValueError(msg)
 
         if pressure_levels is None:
-            pressure_levels = pressure_levels_at_model_levels(20_000.0, 50_000.0)
+            pressure_levels = mlmod.model_level_reference_pressure(20_000.0, 50_000.0)
         self.pressure_levels = metsource.parse_pressure_levels(pressure_levels)
         self.variables = metsource.parse_variables(variables, self.pressure_level_variables)
 
@@ -334,7 +329,6 @@ class HRESModelLevel(ECMWFAPI):
         xr_kwargs: dict[str, Any] | None = None,
         **kwargs: Any,
     ) -> MetDataset:
-
         if dataset:
             msg = "Parameter 'dataset' is not supported for Model-level ERA5 data"
             raise ValueError(msg)
@@ -376,21 +370,22 @@ class HRESModelLevel(ECMWFAPI):
         date = self.forecast_time.strftime("%Y-%m-%d")
         time = self.forecast_time.strftime("%H:%M:%S")
         steps = self.get_forecast_steps(times)
-        # param 152 = log surface pressure, needed for metview level conversion
-        grib_params = set((*self.variable_ecmwfids, 152))
+        # param 152 = log surface pressure, needed for model level conversion
+        grib_params = {*self.variable_ecmwfids, 152}
         return (
             f"retrieve,\n"
             f"class=od,\n"
             f"date={date},\n"
             f"expver=1,\n"
-            f"levelist={'/'.join(str(lev) for lev in sorted(self.levels))},\n"
+            f"levelist={'/'.join(str(lev) for lev in sorted(self.model_levels))},\n"
             f"levtype=ml,\n"
             f"param={'/'.join(str(p) for p in sorted(grib_params))},\n"
             f"step={'/'.join(str(s) for s in sorted(steps))},\n"
             f"stream=oper,\n"
             f"time={time},\n"
             f"type=fc,\n"
-            f"grid={self.grid}/{self.grid}"
+            f"grid={self.grid}/{self.grid},\n"
+            "format=netcdf"
         )
 
     def _set_server(self) -> None:
@@ -413,7 +408,7 @@ class HRESModelLevel(ECMWFAPI):
     ) -> None:
         """Download, convert, and cache HRES model level data.
 
-        This function builds a MARS request and retrieves a single GRIB file.
+        This function builds a MARS request and retrieves a single NetCDF file.
         The calling function should ensure that all times will be contained
         in a single file on tape in the MARS archive.
 
@@ -421,7 +416,7 @@ class HRESModelLevel(ECMWFAPI):
         retrieved data will include the Cartesian product of all unique
         dates and times in the list of specified times.
 
-        After retrieval, this function processes the GRIB file
+        After retrieval, this function processes the NetCDF file
         to produce the dataset specified by class attributes.
 
         Parameters
@@ -429,67 +424,36 @@ class HRESModelLevel(ECMWFAPI):
         times : list[datetime]
             Times to download in a single MARS request.
 
-        Notes
-        -----
-        This function depends on `metview <https://metview.readthedocs.io/en/latest/python.html>`_
-        python bindings and binaries.
-
-        The lifetime of the metview import must last until processed datasets are cached
-        to avoid premature deletion of metview temporary files.
         """
-        try:
-            import metview as mv
-        except ModuleNotFoundError as exc:
-            dependencies.raise_module_not_found_error(
-                "model_level.grib_to_dataset function",
-                package_name="metview",
-                module_not_found_error=exc,
-                extra="See https://metview.readthedocs.io/en/latest/install.html for instructions.",
-            )
-        except ImportError as exc:
-            msg = "Failed to import metview"
-            raise ImportError(msg) from exc
-
         if self.cachestore is None:
             msg = "Cachestore is required to download and cache data"
             raise ValueError(msg)
 
-        stack = contextlib.ExitStack()
         request = self.mars_request(times)
 
-        if not self.cache_grib:
+        stack = contextlib.ExitStack()
+        if not self.cache_download:
             target = stack.enter_context(temp.temp_file())
         else:
             name = hashlib.md5(request.encode()).hexdigest()
-            target = self.cachestore.path(f"hresml-{name}.grib")
+            target = self.cachestore.path(f"hresml-{name}.nc")
 
         with stack:
-            if not self.cache_grib or not self.cachestore.exists(target):
+            if not self.cache_download or not self.cachestore.exists(target):
                 if not hasattr(self, "server"):
                     self._set_server()
                 self.server.execute(request, target)
 
-            # Read contents of GRIB file as metview Fieldset
-            LOG.debug("Opening GRIB file")
-            fs_ml = mv.read(target)
-
-            # reduce memory overhead by caching one timestep at a time
-            for time, step in zip(times, self.get_forecast_steps(times), strict=True):
-                fs_pl = mv.Fieldset()
-                selection = dict(step=step)
-                lnsp = fs_ml.select(shortName="lnsp", **selection)
-                for var in self.variables:
-                    LOG.debug(
-                        f"Converting {var.short_name} at {time.strftime('%Y-%m-%d %H:%M:%S')}"
-                        + f" (step {step})"
-                    )
-                    f_ml = fs_ml.select(shortName=var.short_name, **selection)
-                    f_pl = mv.mvl_ml2hPa(lnsp, f_ml, self.pressure_levels)
-                    fs_pl = mv.merge(fs_pl, f_pl)
-
-                # Create, validate, and cache dataset
-                ds = fs_pl.to_dataset()
-                ds = ds.rename(isobaricInhPa="level", time="initialization_time")
-                ds = ds.rename(step="time").assign_coords(time=time).expand_dims("time")
-                ds.attrs["pycontrails_version"] = pycontrails.__version__
-                self.cache_dataset(ds)
+            LOG.debug("Opening model level data file")
+
+            # Use a chunking scheme harmonious with self.cache_dataset, which groups by time
+            # Because ds_ml is dask-backed, nothing gets computed until cache_dataset is called
+            ds_ml = xr.open_dataset(target).chunk(time=1)
+
+            ds_ml = ds_ml.rename(level="model_level")
+            lnsp = ds_ml["lnsp"].sel(model_level=1)
+            ds_ml = ds_ml.drop_vars("lnsp")
+
+            ds = mlmod.ml_to_pl(ds_ml, target_pl=self.pressure_levels, lnsp=lnsp)
+            ds.attrs["pycontrails_version"] = pycontrails.__version__
+            self.cache_dataset(ds)