cloudnetpy-qc 1.24.3__tar.gz → 1.25.1__tar.gz

{cloudnetpy_qc-1.24.3/cloudnetpy_qc.egg-info → cloudnetpy_qc-1.25.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnetpy_qc
-Version: 1.24.3
+Version: 1.25.1
 Summary: Quality control routines for CloudnetPy products
 Author-email: Finnish Meteorological Institute <actris-cloudnet@fmi.fi>
 License: MIT License
@@ -80,6 +80,7 @@ print(json_object)
 - `timestamp`: UTC timestamp of the test
 - `qcVersion`: `cloudnetpy-qc` version
 - `tests`: `Test[]`
+- `data_coverage`: float
 
 ### `Test`
 

{cloudnetpy_qc-1.24.3 → cloudnetpy_qc-1.25.1}/README.md

@@ -27,6 +27,7 @@ print(json_object)
 - `timestamp`: UTC timestamp of the test
 - `qcVersion`: `cloudnetpy-qc` version
 - `tests`: `Test[]`
+- `data_coverage`: float
 
 ### `Test`
 

cloudnetpy_qc-1.25.1/cloudnetpy_qc/coverage.py

@@ -0,0 +1,66 @@
+import datetime
+
+import netCDF4
+import numpy as np
+
+from cloudnetpy_qc.variables import Product
+
+RESOLUTIONS = {
+    Product.DISDROMETER: datetime.timedelta(minutes=1),
+    Product.L3_CF: datetime.timedelta(hours=1),
+    Product.L3_IWC: datetime.timedelta(hours=1),
+    Product.L3_LWC: datetime.timedelta(hours=1),
+    Product.MWR: datetime.timedelta(minutes=5),
+    Product.MWR_MULTI: datetime.timedelta(minutes=30),
+    Product.MWR_SINGLE: datetime.timedelta(minutes=5),
+    Product.WEATHER_STATION: datetime.timedelta(minutes=10),
+    Product.RAIN_GAUGE: datetime.timedelta(minutes=1),
+    Product.DOPPLER_LIDAR_WIND: datetime.timedelta(hours=1.5),
+}
+DEFAULT_RESOLUTION = datetime.timedelta(seconds=30)
+
+
+def data_coverage(
+    nc: netCDF4.Dataset,
+) -> tuple[float, datetime.timedelta, datetime.timedelta] | None:
+    time = np.array(nc["time"][:])
+    time_unit = datetime.timedelta(hours=1)
+    try:
+        n_time = len(time)
+    except (TypeError, ValueError):
+        return None
+    if n_time < 2:
+        return None
+    if nc.cloudnet_file_type == "model":
+        expected_res = _model_resolution(nc)
+    else:
+        product = Product(nc.cloudnet_file_type)
+        expected_res = RESOLUTIONS.get(product, DEFAULT_RESOLUTION)
+    duration = get_duration(nc)
+    bins = max(1, duration // expected_res)
+    hist, _ = np.histogram(time, bins=bins, range=(0, duration / time_unit))
+    coverage = np.count_nonzero(hist > 0) / len(hist)
+    actual_res = np.median(np.diff(time)) * time_unit
+    return coverage, expected_res, actual_res
+
+
+def _model_resolution(nc: netCDF4.Dataset) -> datetime.timedelta:
+    source = nc.source.lower()
+    if "gdas" in source or "ecmwf open" in source:
+        return datetime.timedelta(hours=3)
+    return datetime.timedelta(hours=1)
+
+
+def get_duration(nc: netCDF4.Dataset) -> datetime.timedelta:
+    now = datetime.datetime.now(tz=datetime.timezone.utc)
+    if now.date() == _get_date(nc):
+        midnight = now.replace(hour=0, minute=0, second=0, microsecond=0)
+        duration = now - midnight
+    else:
+        duration = datetime.timedelta(days=1)
+    return duration
+
+
+def _get_date(nc: netCDF4.Dataset) -> datetime.date:
+    date_in_file = [int(getattr(nc, x)) for x in ("year", "month", "day")]
+    return datetime.date(*date_in_file)

{cloudnetpy_qc-1.24.3 → cloudnetpy_qc-1.25.1}/cloudnetpy_qc/quality.py

@@ -14,9 +14,12 @@ from typing import NamedTuple, TypedDict
 import netCDF4
 import numpy as np
 import scipy.stats
+from cftime import num2pydate
 from numpy import ma
 from requests import RequestException
 
+from cloudnetpy_qc.coverage import data_coverage, get_duration
+
 from . import utils
 from .variables import LEVELS, VARIABLES, Product
 from .version import __version__
@@ -53,6 +56,7 @@ class FileReport(NamedTuple):
     timestamp: datetime.datetime
     qc_version: str
     tests: list[TestReport]
+    data_coverage: float | None
 
     def to_dict(self) -> dict:
         return {
@@ -72,8 +76,9 @@ class FileReport(NamedTuple):
 
 
 class SiteMeta(TypedDict):
-    latitude: float | None
-    longitude: float | None
+    time: np.ndarray | None
+    latitude: float | np.ndarray | None
+    longitude: float | np.ndarray | None
     altitude: float | None
 
 
@@ -84,6 +89,7 @@ def run_tests(
     ignore_tests: list[str] | None = None,
 ) -> FileReport:
     filename = Path(filename)
+    coverage = None
     if isinstance(product, str):
         product = Product(product)
     with netCDF4.Dataset(filename) as nc:
@@ -110,11 +116,15 @@ def run_tests(
                 test_instance._add_error(
                     f"Failed to run test: {err} ({type(err).__name__})"
                 )
+                logging.exception("Failed to run test:")
             test_reports.append(test_instance.report)
+            if test_instance.coverage is not None:
+                coverage = test_instance.coverage
     return FileReport(
         timestamp=datetime.datetime.now(tz=datetime.timezone.utc),
         qc_version=__version__,
         tests=test_reports,
+        data_coverage=coverage,
     )
 
 
@@ -124,6 +134,7 @@ class Test:
     name: str
     description: str
     products: Iterable[Product] = Product.all()
+    coverage: float | None = None
 
     def __init__(
         self, nc: netCDF4.Dataset, filename: Path, product: Product, site_meta: SiteMeta
@@ -190,19 +201,6 @@ class Test:
                     )
                     self._add_warning(msg)
 
-    def _get_date(self):
-        date_in_file = [int(getattr(self.nc, x)) for x in ("year", "month", "day")]
-        return datetime.date(*date_in_file)
-
-    def _get_duration(self) -> datetime.timedelta:
-        now = datetime.datetime.now(tz=datetime.timezone.utc)
-        if now.date() == self._get_date():
-            midnight = now.replace(hour=0, minute=0, second=0, microsecond=0)
-            duration = now - midnight
-        else:
-            duration = datetime.timedelta(days=1)
-        return duration
-
 
 # --------------------#
 # ------ Infos ------ #
@@ -287,45 +285,12 @@ class TestDataCoverage(Test):
     name = "Data coverage"
     description = "Test that file contains enough data."
 
-    RESOLUTIONS = {
-        Product.DISDROMETER: datetime.timedelta(minutes=1),
-        Product.L3_CF: datetime.timedelta(hours=1),
-        Product.L3_IWC: datetime.timedelta(hours=1),
-        Product.L3_LWC: datetime.timedelta(hours=1),
-        Product.MWR: datetime.timedelta(minutes=5),
-        Product.MWR_MULTI: datetime.timedelta(minutes=30),
-        Product.MWR_SINGLE: datetime.timedelta(minutes=5),
-        Product.WEATHER_STATION: datetime.timedelta(minutes=10),
-        Product.RAIN_GAUGE: datetime.timedelta(minutes=1),
-        Product.DOPPLER_LIDAR_WIND: datetime.timedelta(hours=1.5),
-    }
-    DEFAULT_RESOLUTION = datetime.timedelta(seconds=30)
-
-    def _model_resolution(self):
-        source = self.nc.source.lower()
-        if "gdas" in source or "ecmwf open" in source:
-            return datetime.timedelta(hours=3)
-        return datetime.timedelta(hours=1)
-
     def run(self):
-        time = np.array(self.nc["time"][:])
-        time_unit = datetime.timedelta(hours=1)
-        try:
-            n_time = len(time)
-        except (TypeError, ValueError):
-            return
-        if n_time < 2:
+        coverage, expected_res, actual_res = data_coverage(self.nc)
+        if coverage is None:
             return
-        if self.nc.cloudnet_file_type == "model":
-            expected_res = self._model_resolution()
-        else:
-            expected_res = self.RESOLUTIONS.get(self.product, self.DEFAULT_RESOLUTION)
-        duration = self._get_duration()
-        bins = max(1, duration // expected_res)
-        hist, _bin_edges = np.histogram(
-            time, bins=bins, range=(0, duration / time_unit)
-        )
-        missing = np.count_nonzero(hist == 0) / len(hist) * 100
+        self.coverage = coverage
+        missing = (1 - coverage) * 100
         if missing > 20:
             message = f"{round(missing)}% of day's data is missing."
             if missing > 60:
@@ -333,7 +298,6 @@ class TestDataCoverage(Test):
             else:
                 self._add_info(message)
 
-        actual_res = np.median(np.diff(time)) * time_unit
         if actual_res > expected_res * 1.05:
             self._add_warning(
                 f"Expected a measurement with interval at least {expected_res},"
@@ -789,7 +753,7 @@ class TestModelData(Test):
         if n_time < 2:
             return
 
-        duration = self._get_duration()
+        duration = get_duration(self.nc)
         should_be_data_until = duration / time_unit
 
         for key in ("temperature", "pressure", "q"):
@@ -838,14 +802,27 @@ class TestCoordinates(Test):
                 self._add_error(f"Variable '{key}' is missing")
 
         if "latitude" in self.nc.variables and "longitude" in self.nc.variables:
-            site_lat = self.site_meta["latitude"]
-            site_lon = self.site_meta["longitude"]
+            site_lat = np.atleast_1d(self.site_meta["latitude"])
+            site_lon = np.atleast_1d(self.site_meta["longitude"])
             file_lat = np.atleast_1d(self.nc["latitude"][:])
             file_lon = np.atleast_1d(self.nc["longitude"][:])
             file_lon[file_lon > 180] -= 360
-            dist = utils.haversine(site_lat, site_lon, file_lat, file_lon)
+
+            if self.site_meta.get("time") and file_lat.size > 1 and file_lon.size > 1:
+                site_time = self._read_site_time()
+                file_time = self._read_file_time()
+                idx = utils.find_closest(file_time, site_time)
+                file_lat = file_lat[idx]
+                file_lon = file_lon[idx]
+            else:
+                file_lat, file_lon = utils.average_coordinate(file_lat, file_lon)
+                site_lat, site_lon = utils.average_coordinate(site_lat, site_lon)
+
+            dist = np.atleast_1d(
+                utils.haversine(site_lat, site_lon, file_lat, file_lon)
+            )
             i = np.argmax(dist)
-            max_dist = 100 if self.nc.cloudnet_file_type == "model" else 10
+            max_dist = self._calc_max_dist(site_lat, site_lon)
             if dist[i] > max_dist:
                 self._add_error(
                     f"Variables 'latitude' and 'longitude' do not match "
@@ -867,6 +844,37 @@ class TestCoordinates(Test):
                     f"but received {round(file_alt[i])}\u00a0m"
                 )
 
+    def _read_site_time(self):
+        for dt in self.site_meta["time"]:
+            if (
+                not isinstance(dt, datetime.datetime)
+                or dt.tzinfo is None
+                or dt.tzinfo.utcoffset(dt) is None
+            ):
+                raise ValueError("Naive datetimes are not supported")
+        naive_dt = [
+            dt.astimezone(datetime.timezone.utc).replace(tzinfo=None)
+            for dt in self.site_meta["time"]
+        ]
+        return np.array(naive_dt, dtype="datetime64[s]")
+
+    def _read_file_time(self):
+        naive_dt = num2pydate(
+            self.nc["time"][:], self.nc["time"].units, self.nc["time"].calendar
+        )
+        return np.array(naive_dt, dtype="datetime64[s]")
+
+    def _calc_max_dist(self, latitude, longitude):
+        if self.nc.cloudnet_file_type == "model":
+            angle = 1  # Model resolution should be at least 1 degrees.
+            half_angle = angle / 2
+            min_lat = np.maximum(-90, latitude - half_angle)
+            max_lat = np.minimum(90, latitude + half_angle)
+            min_lon = np.maximum(-180, longitude - half_angle)
+            max_lon = np.minimum(180, longitude + half_angle)
+            return utils.haversine(min_lat, min_lon, max_lat, max_lon)
+        return 10
+
 
 # ------------------------------#
 # ------ Error / Warning ------ #

{cloudnetpy_qc-1.24.3 → cloudnetpy_qc-1.25.1}/cloudnetpy_qc/utils.py

@@ -141,3 +141,51 @@ def haversine(
 
     a = np.sin(dlat / 2) ** 2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon / 2) ** 2
     return 2 * r * np.arcsin(np.sqrt(a))
+
+
+def find_closest(x: npt.NDArray, x_new: npt.NDArray) -> npt.NDArray[np.intp]:
+    """Find the closest values between two arrays.
+
+    Args:
+        x: Sorted array.
+        x_new: Sorted array.
+
+    Returns:
+        Indices into `x` which correspond to the closest values in `x_new`.
+
+    Example:
+        >>> x = np.array([0.9, 1.2, 2.0, 2.1])
+        >>> x_new = np.array([1, 2])
+        >>> find_closest(x, x_new)
+        array([0, 2])
+    """
+    idx = np.searchsorted(x, x_new)
+    idx_left = np.clip(idx - 1, 0, len(x) - 1)
+    idx_right = np.clip(idx, 0, len(x) - 1)
+    diff_left = np.abs(x_new - x[idx_left])
+    diff_right = np.abs(x_new - x[idx_right])
+    return np.where(diff_left < diff_right, idx_left, idx_right)
+
+
+def average_coordinate(
+    latitude: npt.NDArray, longitude: npt.NDArray
+) -> tuple[float, float]:
+    """Calculate average position from given coordinates.
+
+    Args:
+        latitude: Array of latitudes.
+        longitude: Array of longitudes.
+
+    Returns:
+        Tuple of average latitude and longitude.
+    """
+    if latitude.size == longitude.size == 1:
+        return latitude[0], longitude[0]
+    latitude = np.radians(latitude)
+    longitude = np.radians(longitude)
+    x = np.mean(np.cos(latitude) * np.cos(longitude))
+    y = np.mean(np.cos(latitude) * np.sin(longitude))
+    z = np.mean(np.sin(latitude))
+    avg_lat = np.degrees(np.atan2(z, np.sqrt(x * x + y * y)))
+    avg_lon = np.degrees(np.atan2(y, x))
+    return avg_lat, avg_lon

{cloudnetpy_qc-1.24.3 → cloudnetpy_qc-1.25.1}/cloudnetpy_qc/version.py

@@ -1,6 +1,6 @@
 """Cloudnetpy-QC version."""
 
 MAJOR = 1
-MINOR = 24
-PATCH = 3
+MINOR = 25
+PATCH = 1
 __version__ = f"{MAJOR}.{MINOR}.{PATCH}"

{cloudnetpy_qc-1.24.3 → cloudnetpy_qc-1.25.1/cloudnetpy_qc.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnetpy_qc
-Version: 1.24.3
+Version: 1.25.1
 Summary: Quality control routines for CloudnetPy products
 Author-email: Finnish Meteorological Institute <actris-cloudnet@fmi.fi>
 License: MIT License
@@ -80,6 +80,7 @@ print(json_object)
 - `timestamp`: UTC timestamp of the test
 - `qcVersion`: `cloudnetpy-qc` version
 - `tests`: `Test[]`
+- `data_coverage`: float
 
 ### `Test`
 

{cloudnetpy_qc-1.24.3 → cloudnetpy_qc-1.25.1}/cloudnetpy_qc.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@ MANIFEST.in
 README.md
 pyproject.toml
 cloudnetpy_qc/__init__.py
+cloudnetpy_qc/coverage.py
 cloudnetpy_qc/py.typed
 cloudnetpy_qc/quality.py
 cloudnetpy_qc/utils.py

{cloudnetpy_qc-1.24.3 → cloudnetpy_qc-1.25.1}/pyproject.toml

@@ -30,7 +30,7 @@ Repository = "https://github.com/actris-cloudnet/cloudnetpy-qc"
 Changelog = "https://github.com/actris-cloudnet/cloudnetpy-qc/blob/main/CHANGELOG.md"
 
 [[tool.mypy.overrides]]
-module = ["cfchecker.*", "scipy.*"]
+module = ["cfchecker.*", "cftime.*", "scipy.*"]
 ignore_missing_imports = true
 
 [tool.release-version]

{cloudnetpy_qc-1.24.3 → cloudnetpy_qc-1.25.1}/tests/test_qc.py

@@ -93,6 +93,7 @@ class Check:
     def __init__(self, filename: str, file_type: str | None = None):
         # Norunda
         site_meta: quality.SiteMeta = {
+            "time": None,
             "latitude": 60.086,
             "longitude": 17.479,
             "altitude": 46.0,