cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

cloudnetpy/instruments/copernicus.py

@@ -1,22 +1,27 @@
 """Module for reading raw cloud radar data."""
+
+import datetime
 import os
+import tempfile
+from os import PathLike
 from tempfile import TemporaryDirectory
+from uuid import UUID
 
 import numpy as np
 
 from cloudnetpy import concat_lib, output, utils
 from cloudnetpy.instruments.instruments import COPERNICUS
 from cloudnetpy.instruments.nc_radar import ChilboltonRadar
-from cloudnetpy.metadata import MetaData
+from cloudnetpy.metadata import COMMON_ATTRIBUTES, MetaData
 
 
 def copernicus2nc(
-    raw_files: str,
-    output_file: str,
+    raw_files: str | PathLike,
+    output_file: str | PathLike,
     site_meta: dict,
-    uuid: str | None = None,
-    date: str | None = None,
-) -> str:
+    uuid: str | UUID | None = None,
+    date: str | datetime.date | None = None,
+) -> UUID:
     """Converts 'Copernicus' cloud radar data into Cloudnet Level 1b netCDF file.
 
     Args:
@@ -41,6 +46,10 @@ def copernicus2nc(
           >>> copernicus2nc('/one/day/of/copernicus/files/', 'radar.nc', site_meta)
 
     """
+    if isinstance(date, str):
+        date = datetime.date.fromisoformat(date)
+    uuid = utils.get_uuid(uuid)
+
     keymap = {
         "ZED_HC": "Zh",
         "VEL_HC": "v",
@@ -55,15 +64,25 @@ def copernicus2nc(
         "beamwidthH": "beamwidthH",
     }
 
+    nc_filename: str | PathLike
     with TemporaryDirectory() as temp_dir:
         if os.path.isdir(raw_files):
-            nc_filename = f"{temp_dir}/tmp.nc"
-            valid_filenames = utils.get_sorted_filenames(raw_files, ".nc")
-            valid_filenames = utils.get_files_with_common_range(valid_filenames)
-            variables = list(keymap.keys())
-            concat_lib.concatenate_files(
-                valid_filenames, nc_filename, variables=variables
-            )
+            with tempfile.NamedTemporaryFile(
+                dir=temp_dir,
+                suffix=".nc",
+                delete=False,
+            ) as temp_file:
+                nc_filename = temp_file.name
+                valid_filenames = utils.get_sorted_filenames(raw_files, ".nc")
+                valid_filenames = utils.get_files_with_variables(
+                    valid_filenames, ["time", "ZED_HC"]
+                )
+                variables = list(keymap.keys())
+                concat_lib.concatenate_files(
+                    valid_filenames,
+                    nc_filename,
+                    variables=variables,
+                )
         else:
             nc_filename = raw_files
 
@@ -75,19 +94,26 @@ def copernicus2nc(
             copernicus.sort_timestamps()
             copernicus.remove_duplicate_timestamps()
             copernicus.calibrate_reflectivity()
-            copernicus.screen_by_snr(snr_limit=3)
+            copernicus.screen_using_top_gates_snr()
             copernicus.mask_corrupted_values()
+            copernicus.mask_first_range_gates()
             copernicus.mask_invalid_data()
-            copernicus.add_time_and_range()
+            copernicus.fix_range_offset(site_meta)
+            copernicus.screen_negative_ranges()
             copernicus.add_radar_specific_variables()
             copernicus.add_nyquist_velocity(keymap)
             copernicus.add_site_geolocation()
-            valid_indices = copernicus.add_zenith_and_azimuth_angles()
+            valid_indices = copernicus.add_zenith_and_azimuth_angles(
+                elevation_threshold=1.1,
+                elevation_diff_threshold=0.5,
+                azimuth_diff_threshold=0.1,
+            )
             copernicus.screen_time_indices(valid_indices)
             copernicus.add_height()
+            copernicus.test_if_all_masked()
         attributes = output.add_time_attribute(ATTRIBUTES, copernicus.date)
         output.update_attributes(copernicus.data, attributes)
-        uuid = output.save_level1b(copernicus, output_file, uuid)
+        output.save_level1b(copernicus, output_file, uuid)
         return uuid
 
 
@@ -100,17 +126,35 @@ class Copernicus(ChilboltonRadar):
 
     """
 
-    def __init__(self, full_path: str, site_meta: dict):
+    def __init__(self, full_path: str | PathLike, site_meta: dict) -> None:
         super().__init__(full_path, site_meta)
         self.instrument = COPERNICUS
 
-    def calibrate_reflectivity(self):
-        default_offset = -146.8  # TODO: check this value
-        calibration_factor = self.site_meta.get("calibration_offset", default_offset)
-        self.data["Zh"].data[:] += calibration_factor
-        self.append_data(np.array(calibration_factor), "calibration_offset")
+    def calibrate_reflectivity(self) -> None:
+        zed_hc = self.dataset["ZED_HC"]
+        offset_applied = getattr(zed_hc, "applied_calibration_offset", 0)
+
+        # Estimated by comparing with MIRA-35 data:
+        default_offset = -149.5
+        zh_offset = self.site_meta.get("Zh_offset", default_offset)
+
+        self.data["Zh"].data[:] = self.data["Zh"].data[:] - offset_applied + zh_offset
+        self.append_data(np.array(zh_offset, dtype=np.float32), "Zh_offset")
+
+    def fix_range_offset(self, site_meta: dict) -> None:
+        range_var = self.dataset["range"]
+        offset_applied = getattr(range_var, "range_offset", 0)
 
-    def mask_corrupted_values(self):
+        # Estimated by comparing with MIRA-35 data:
+        default_offset = -720
+        range_offset = site_meta.get("range_offset", default_offset)
+
+        self.data["range"].data[:] = (
+            self.data["range"].data[:] - offset_applied + range_offset
+        )
+        self.append_data(np.array(range_offset, dtype=float), "range_offset")
+
+    def mask_corrupted_values(self) -> None:
         """Experimental masking of corrupted Copernicus data.
 
         Notes:
@@ -122,14 +166,48 @@ class Copernicus(ChilboltonRadar):
             ind = np.where(np.abs(self.data[key][:]) > value)
             self.data["v"].mask_indices(ind)
 
+    def screen_negative_ranges(self) -> None:
+        """Screens negative range values."""
+        valid_ind = np.where(self.data["range"][:] >= 0)[0]
+        for key, cloudnet_array in self.data.items():
+            try:
+                data = cloudnet_array[:]
+                if data.ndim == 2:
+                    cloudnet_array.data = data[:, valid_ind]
+                elif key == "range":
+                    cloudnet_array.data = data[valid_ind]
+            except IndexError:
+                continue
+
 
 ATTRIBUTES = {
-    "calibration_offset": MetaData(
+    "Zh_offset": MetaData(
         long_name="Radar reflectivity calibration offset",
-        units="1",
-        comment="Calibration offset applied.",
+        units="dBZ",
+        comment=(
+            "Calibration offset applied after removing the original offset "
+            "from the raw files."
+        ),
+        dimensions=None,
+    ),
+    "Zh": COMMON_ATTRIBUTES["Zh"]._replace(ancillary_variables="Zh_offset"),
+    "range_offset": MetaData(
+        long_name="Radar range offset",
+        units="m",
+        comment=(
+            "Range offset applied after removing the original offset "
+            "from the raw files."
+        ),
+        dimensions=None,
+    ),
+    "range": COMMON_ATTRIBUTES["range"]._replace(ancillary_variables="range_offset"),
+    "antenna_diameter": MetaData(
+        long_name="Antenna diameter", units="m", dimensions=("time",)
+    ),
+    "beamwidthV": MetaData(
+        long_name="Vertical angular beamwidth", units="degree", dimensions=("time",)
+    ),
+    "beamwidthH": MetaData(
+        long_name="Horizontal angular beamwidth", units="degree", dimensions=("time",)
     ),
-    "antenna_diameter": MetaData(long_name="Antenna diameter", units="m"),
-    "beamwidthV": MetaData(long_name="Vertical angular beamwidth", units="degree"),
-    "beamwidthH": MetaData(long_name="Horizontal angular beamwidth", units="degree"),
 }

cloudnetpy/instruments/da10.py

@@ -0,0 +1,54 @@
+import datetime
+import logging
+from os import PathLike
+
+import netCDF4
+
+from cloudnetpy import utils
+from cloudnetpy.exceptions import LidarDataError
+from cloudnetpy.instruments import instruments
+from cloudnetpy.instruments.nc_lidar import NcLidar
+
+
+class Da10(NcLidar):
+    """Class for Vaisala DA10 differential absorption lidar."""
+
+    def __init__(
+        self,
+        file_name: str | PathLike,
+        site_meta: dict,
+        expected_date: datetime.date | None = None,
+    ) -> None:
+        super().__init__()
+        self.file_name = file_name
+        self.site_meta = site_meta
+        self.expected_date = expected_date
+        self.instrument = instruments.DA10
+
+    def read_ceilometer_file(self, calibration_factor: float | None = None) -> None:
+        with netCDF4.Dataset(self.file_name) as dataset:
+            self.dataset = dataset
+            self._fetch_attributes()
+            self._fetch_zenith_angle("tilt_angle", default=3.0)
+            self._fetch_range(reference="lower")
+            self._fetch_lidar_variables(calibration_factor)
+            self._fetch_time_and_date()
+            self.dataset = None
+
+    def _fetch_lidar_variables(self, calibration_factor: float | None = None) -> None:
+        if self.dataset is None:
+            msg = "No dataset found"
+            raise RuntimeError(msg)
+        beta_raw = self.dataset.variables["beta_att"][:]
+        if utils.is_all_masked(beta_raw):
+            msg = "All beta_raw values are masked. Check the input file(s)."
+            raise LidarDataError(msg)
+        if calibration_factor is None:
+            logging.warning("Using default calibration factor")
+            calibration_factor = 1
+        beta_raw *= calibration_factor
+        self.data["calibration_factor"] = float(calibration_factor)
+        self.data["beta_raw"] = beta_raw
+
+    def _fetch_attributes(self) -> None:
+        self.serial_number = getattr(self.dataset, "instrument_serial_number", None)