cloudnetpy 1.50.0__tar.gz → 1.51.1__tar.gz

{cloudnetpy-1.50.0/cloudnetpy.egg-info → cloudnetpy-1.51.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudnetpy
-Version: 1.50.0
+Version: 1.51.1
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License

cloudnetpy-1.51.1/cloudnetpy/instruments/campbell_scientific.py

@@ -0,0 +1,135 @@
+import binascii
+import re
+from datetime import datetime
+from typing import NamedTuple
+
+import numpy as np
+
+from cloudnetpy import utils
+from cloudnetpy.exceptions import InconsistentDataError, ValidTimeStampError
+from cloudnetpy.instruments import instruments
+from cloudnetpy.instruments.ceilometer import Ceilometer
+
+
+class Cs135(Ceilometer):
+    def __init__(
+        self, full_path: str, site_meta: dict, expected_date: str | None = None
+    ):
+        super().__init__()
+        self.full_path = full_path
+        self.site_meta = site_meta
+        self.expected_date = expected_date
+        self.data = {}
+        self.metadata = {}
+        self.instrument = instruments.CS135
+
+    def read_ceilometer_file(self, calibration_factor: float | None = None) -> None:
+        with open(self.full_path, mode="rb") as f:
+            content = f.read()
+        timestamps = []
+        profiles = []
+        tilt_angles = []
+        range_resolutions = []
+
+        parts = re.split(rb"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{6}),", content)
+        for i in range(1, len(parts), 2):
+            timestamp = datetime.strptime(parts[i].decode(), "%Y-%m-%dT%H:%M:%S.%f")
+            try:
+                self._check_timestamp(timestamp)
+            except ValidTimeStampError:
+                continue
+            try:
+                message = _read_message(parts[i + 1])
+            except InvalidMessageError:
+                continue
+            profile = (message.data[:-2] * 1e-8) * (message.scale / 100)
+            timestamps.append(timestamp)
+            profiles.append(profile)
+            tilt_angles.append(message.tilt_angle)
+            range_resolutions.append(message.range_resolution)
+
+        if len(timestamps) == 0:
+            raise ValidTimeStampError("No valid timestamps found in the file")
+        range_resolution = range_resolutions[0]
+        n_gates = len(profiles[0])
+        if any(res != range_resolution for res in range_resolutions):
+            raise InconsistentDataError("Inconsistent range resolution")
+        if any(len(profile) != n_gates for profile in profiles):
+            raise InconsistentDataError("Inconsistent number of gates")
+
+        self.data["beta_raw"] = np.array(profiles)
+        if calibration_factor is None:
+            calibration_factor = 1.0
+        self.data["beta_raw"] *= calibration_factor
+        self.data["calibration_factor"] = calibration_factor
+        self.data["range"] = (
+            np.arange(n_gates) * range_resolution + range_resolution / 2
+        )
+        self.data["time"] = utils.datetime2decimal_hours(timestamps)
+        self.data["zenith_angle"] = np.median(tilt_angles)
+
+    def _check_timestamp(self, timestamp: datetime):
+        timestamp_components = str(timestamp.date()).split("-")
+        if self.expected_date is not None:
+            if timestamp_components != self.expected_date.split("-"):
+                raise ValidTimeStampError
+        if not self.date:
+            self.date = timestamp_components
+        assert timestamp_components == self.date
+
+
+class Message(NamedTuple):
+    scale: int
+    range_resolution: int
+    laser_pulse_energy: int
+    laser_temperature: int
+    tilt_angle: int
+    background_light: int
+    pulse_quantity: int
+    sample_rate: int
+    data: np.ndarray
+
+
+class InvalidMessageError(Exception):
+    pass
+
+
+def _read_message(message: bytes) -> Message:
+    end_idx = message.index(3)
+    content = message[1 : end_idx + 1]
+    expected_checksum = int(message[end_idx + 1 : end_idx + 5], 16)
+    actual_checksum = _crc16(content)
+    if expected_checksum != actual_checksum:
+        raise InvalidMessageError(
+            "Invalid checksum: "
+            f"expected {expected_checksum:04x}, "
+            f"got {actual_checksum:04x}"
+        )
+    lines = message.splitlines()
+    if len(lines[0]) != 11:
+        raise NotImplementedError("Unknown message format")
+    if (msg_no := lines[0][-4:-1]) != b"002":
+        raise NotImplementedError(f"Message number {msg_no.decode()} not implemented")
+    if len(lines) != 5:
+        raise InvalidMessageError("Invalid line count")
+    scale, res, n, energy, lt, ti, bl, pulse, rate, _sum = map(int, lines[2].split())
+    data = _read_backscatter(lines[3].strip(), n)
+    return Message(scale, res, energy, lt, ti, bl, pulse, rate, data)
+
+
+def _read_backscatter(data: bytes, n_gates: int) -> np.ndarray:
+    """Read backscatter values from hex-encoded two's complement values."""
+    n_chars = 5
+    n_bits = n_chars * 4
+    limit = (1 << (n_bits - 1)) - 1
+    offset = 1 << n_bits
+    out = np.array(
+        [int(data[i : i + n_chars], 16) for i in range(0, n_gates * n_chars, n_chars)]
+    )
+    out[out > limit] -= offset
+    return out
+
+
+def _crc16(data: bytes) -> int:
+    """Compute checksum similar to CRC-16-CCITT."""
+    return binascii.crc_hqx(data, 0xFFFF) ^ 0xFFFF

{cloudnetpy-1.50.0 → cloudnetpy-1.51.1}/cloudnetpy/instruments/ceilometer.py

@@ -21,6 +21,8 @@ class NoiseParam:
 class Ceilometer:
     """Base class for all types of ceilometers and pollyxt."""
 
+    serial_number: str | None
+
     def __init__(self, noise_param: NoiseParam = NoiseParam()):
         self.noise_param = noise_param
         self.data: dict = {}  # Need to contain 'beta_raw', 'range' and 'time'
@@ -29,6 +31,7 @@ class Ceilometer:
         self.site_meta: dict = {}
         self.date: list[str] = []
         self.instrument: Instrument | None = None
+        self.serial_number = None
 
     def calc_screened_product(
         self,

{cloudnetpy-1.50.0 → cloudnetpy-1.51.1}/cloudnetpy/instruments/cl61d.py

@@ -23,7 +23,8 @@ class Cl61d(NcLidar):
         """Reads data and metadata from concatenated Vaisala CL61d netCDF file."""
         with netCDF4.Dataset(self.file_name) as dataset:
             self.dataset = dataset
-            self._fetch_zenith_angle("zenith", default=3.0)
+            self._fetch_attributes()
+            self._fetch_zenith_angle("tilt_angle", default=3.0)
             self._fetch_range(reference="lower")
             self._fetch_lidar_variables(calibration_factor)
             self._fetch_time_and_date()
@@ -41,3 +42,6 @@ class Cl61d(NcLidar):
         self.data["depolarisation"] = (
             self.dataset.variables["x_pol"][:] / self.dataset.variables["p_pol"][:]
         )
+
+    def _fetch_attributes(self):
+        self.serial_number = getattr(self.dataset, "instrument_serial_number", None)

{cloudnetpy-1.50.0 → cloudnetpy-1.51.1}/cloudnetpy/instruments/disdrometer/parsivel.py

@@ -1,5 +1,7 @@
+import csv
 import datetime
 import logging
+import re
 from collections.abc import Callable, Iterator, Sequence
 from itertools import islice
 from pathlib import Path
@@ -169,7 +171,7 @@ class Parsivel(CloudnetInstrument):
                 raise ValueError
 
 
-HEADERS = {
+CSV_HEADERS = {
     "Date": "_date",
     "Time": "_time",
     "Intensity of precipitation (mm/h)": "rainfall_rate",
@@ -190,6 +192,47 @@ HEADERS = {
     "Spectrum": "spectrum",
 }
 
+TOA5_HEADERS = {
+    "TIMESTAMP": "_datetime",
+    "rainIntensity": "rainfall_rate",
+    "rain_intensity": "rainfall_rate",
+    "snowIntensity": "snowfall_rate",
+    "snow_intensity": "snowfall_rate",
+    "accPrec": "_rain_accum",
+    "precipitation": "_rain_accum",
+    "weatherCodeWaWa": "synop_WaWa",
+    "weather_code_wawa": "synop_WaWa",
+    "radarReflectivity": "radar_reflectivity",
+    "radar_reflectivity": "radar_reflectivity",
+    "morVisibility": "visibility",
+    "mor_visibility": "visibility",
+    "kineticEnergy": "kinetic_energy",
+    "kinetic_energy": "kinetic_energy",
+    "signalAmplitude": "sig_laser",
+    "signal_amplitude": "sig_laser",
+    "sensorTemperature": "T_sensor",
+    "sensor_temperature": "T_sensor",
+    "pbcTemperature": "_T_pcb",
+    "pbc_temperature": "_T_pcb",
+    "rightTemperature": "_T_right",
+    "right_temperature": "_T_right",
+    "leftTemperature": "_T_left",
+    "left_temperature": "_T_left",
+    "heatingCurrent": "I_heating",
+    "heating_current": "I_heating",
+    "sensorVoltage": "V_power_supply",
+    "sensor_voltage": "V_power_supply",
+    "sensorStatus": "state_sensor",
+    "sensor_status": "state_sensor",
+    "errorCode": "error_code",
+    "error_code": "error_code",
+    "numberParticles": "n_particles",
+    "number_particles": "n_particles",
+    "N": "number_concentration",
+    "V": "fall_velocity",
+    "spectrum": "spectrum",
+}
+
 TELEGRAM = {
     1: "rainfall_rate",
     2: "_rain_accum",
@@ -325,7 +368,7 @@ PARSERS: dict[str, Callable[[Iterator[str]], Any]] = {
 
 
 def _parse_headers(line: str) -> list[str]:
-    return [HEADERS[header.strip()] for header in line.split(";")]
+    return [CSV_HEADERS[header.strip()] for header in line.split(";")]
 
 
 def _parse_telegram(telegram: Sequence[int | None]) -> list[str]:
@@ -359,6 +402,71 @@ def _read_rows(headers: list[str], rows: list[str]) -> dict[str, list]:
     return result
 
 
+def _read_toa5(filename: Path | str | bytes) -> dict[str, list]:
+    """
+    Read ASCII data from Campbell Scientific datalogger such as CR1000.
+
+    References:
+        CR1000 Measurement and Control System.
+        https://s.campbellsci.com/documents/us/manuals/cr1000.pdf
+    """
+    # pylint: disable=too-many-branches,comparison-with-callable
+    with open(filename, encoding="latin1", errors="ignore") as file:
+        reader = csv.reader(file)
+        _origin_line = next(reader)
+        header_line = next(reader)
+        headers = [
+            TOA5_HEADERS.get(re.sub(r"\(.*", "", field)) for field in header_line
+        ]
+        _units_line = next(reader)
+        _process_line = next(reader)
+        data: dict[str, list] = {header: [] for header in headers if header is not None}
+        n_rows = 0
+        n_invalid_rows = 0
+        for data_line in reader:
+            n_rows += 1
+            scalars: dict[str, datetime.datetime | int | float] = {}
+            arrays: dict[str, list] = {
+                "number_concentration": [],
+                "fall_velocity": [],
+                "spectrum": [],
+            }
+            try:
+                for header, value in zip(headers, data_line):
+                    if header is None:
+                        continue
+                    if header == "_datetime":
+                        scalars[header] = datetime.datetime.strptime(
+                            value, "%Y-%m-%d %H:%M:%S"
+                        )
+                    elif header in ("number_concentration", "fall_velocity"):
+                        arrays[header].append(float(value))
+                    elif header == "spectrum":
+                        arrays[header].append(int(value))
+                    elif PARSERS.get(header) == _parse_int:
+                        scalars[header] = int(value)
+                    elif PARSERS.get(header) == _parse_float:
+                        scalars[header] = float(value)
+            except ValueError:
+                n_invalid_rows += 1
+                continue
+            for header, scalar in scalars.items():
+                data[header].append(scalar)
+            if "spectrum" in headers:
+                data["spectrum"].append(
+                    np.array(arrays["spectrum"], dtype="i2").reshape((32, 32))
+                )
+            if "number_concentration" in headers:
+                data["number_concentration"].append(arrays["number_concentration"])
+            if "fall_velocity" in headers:
+                data["fall_velocity"].append(arrays["fall_velocity"])
+        if n_invalid_rows == n_rows:
+            raise DisdrometerDataError("No valid data in file")
+        if n_invalid_rows > 0:
+            logging.info(f"Skipped {n_invalid_rows} invalid rows")
+        return data
+
+
 def _read_parsivel(
     filename: Path | str | bytes, telegram: Sequence[int | None] | None = None
 ) -> dict[str, np.ndarray]:
@@ -366,7 +474,9 @@ def _read_parsivel(
         lines = file.read().splitlines()
     if not lines:
         raise DisdrometerDataError("File is empty")
-    if "Date" in lines[0]:
+    if "TOA5" in lines[0]:
+        data = _read_toa5(filename)
+    elif "Date" in lines[0]:
         headers = _parse_headers(lines[0])
         data = _read_rows(headers, lines[1:])
     elif telegram is not None:

{cloudnetpy-1.50.0 → cloudnetpy-1.51.1}/cloudnetpy/instruments/lufft.py

@@ -12,8 +12,6 @@ from cloudnetpy.instruments.nc_lidar import NcLidar
 class LufftCeilo(NcLidar):
     """Class for Lufft chm15k ceilometer."""
 
-    serial_number: str | None
-
     def __init__(
         self, file_name: str, site_meta: dict, expected_date: str | None = None
     ):
@@ -21,7 +19,6 @@ class LufftCeilo(NcLidar):
         self.file_name = file_name
         self.site_meta = site_meta
         self.expected_date = expected_date
-        self.serial_number = None
 
     def read_ceilometer_file(self, calibration_factor: float | None = None) -> None:
         """Reads data and metadata from Jenoptik netCDF file."""

{cloudnetpy-1.50.0 → cloudnetpy-1.51.1}/cloudnetpy/instruments/nc_lidar.py

@@ -30,7 +30,7 @@ class NcLidar(Ceilometer):
     def _fetch_zenith_angle(self, key: str, default: float = 3.0) -> None:
         assert self.dataset is not None
         if key in self.dataset.variables:
-            zenith_angle = self.dataset.variables[key][:]
+            zenith_angle = np.median(self.dataset.variables[key][:])
         else:
             zenith_angle = float(default)
             logging.warning(f"No zenith angle found, assuming {zenith_angle} degrees")

{cloudnetpy-1.50.0 → cloudnetpy-1.51.1}/cloudnetpy/instruments/pollyxt.py

@@ -103,6 +103,7 @@ class PollyXt(Ceilometer):
             raise InconsistentDataError(
                 "Inconsistent number of pollyxt bsc / depol files"
             )
+        self._fetch_attributes(bsc_files[0])
         self.data["range"] = _read_array_from_multiple_files(
             bsc_files, depol_files, "height"
         )
@@ -157,6 +158,11 @@ class PollyXt(Ceilometer):
                         return channel
         raise ValidTimeStampError("No functional pollyXT backscatter channels found")
 
+    def _fetch_attributes(self, file: str) -> None:
+        with netCDF4.Dataset(file, "r") as nc:
+            if hasattr(nc, "source"):
+                self.serial_number = nc.source.lower()
+
 
 def _read_array_from_multiple_files(files1: list, files2: list, key) -> np.ndarray:
     array: np.ndarray = np.array([])

{cloudnetpy-1.50.0 → cloudnetpy-1.51.1}/cloudnetpy/plotting/plotting.py

@@ -755,7 +755,7 @@ def plot_2d(
 
 
 def compare_files(
-    nc_files: list,
+    nc_files: tuple[str, str],
     field_name: str,
     show: bool = True,
     relative_err: bool = False,
@@ -790,7 +790,7 @@ def compare_files(
     ax_values = [_read_ax_values(nc_file) for nc_file in nc_files]
     subtitle = (
         f" - {os.path.basename(nc_files[0])}",
-        f" - {os.path.basename(nc_files[0])}",
+        f" - {os.path.basename(nc_files[1])}",
     )
     n_subs = 3 if relative_err is True else 2
     fig, axes = _initialize_figure(n_subs, dpi)

{cloudnetpy-1.50.0 → cloudnetpy-1.51.1}/cloudnetpy/version.py

@@ -1,4 +1,4 @@
 MAJOR = 1
-MINOR = 50
-PATCH = 0
+MINOR = 51
+PATCH = 1
 __version__ = f"{MAJOR}.{MINOR}.{PATCH}"

{cloudnetpy-1.50.0 → cloudnetpy-1.51.1/cloudnetpy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudnetpy
-Version: 1.50.0
+Version: 1.51.1
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License

cloudnetpy-1.50.0/cloudnetpy/instruments/campbell_scientific.py

@@ -1,110 +0,0 @@
-import logging
-import re
-from datetime import datetime
-
-import numpy as np
-
-from cloudnetpy import utils
-from cloudnetpy.exceptions import ValidTimeStampError
-from cloudnetpy.instruments import instruments
-from cloudnetpy.instruments.ceilometer import Ceilometer
-
-
-class Cs135(Ceilometer):
-    def __init__(
-        self, full_path: str, site_meta: dict, expected_date: str | None = None
-    ):
-        super().__init__()
-        self.full_path = full_path
-        self.site_meta = site_meta
-        self.expected_date = expected_date
-        self.data = {}
-        self.metadata = {}
-        self.instrument = instruments.CS135
-
-    def read_ceilometer_file(self, calibration_factor: float | None = None) -> None:
-        with open(self.full_path, mode="r", encoding="utf-8") as f:
-            lines = f.readlines()
-        timestamps, profiles, scales, tilt_angles = [], [], [], []
-        range_resolution, n_gates = 0, 0
-        for i, line in enumerate(lines):
-            line_splat = line.strip().split(",")
-            if is_timestamp(line_splat[0]):
-                timestamp = datetime.strptime(line_splat[0], "%Y-%m-%dT%H:%M:%S.%f")
-                try:
-                    self._check_timestamp(timestamp)
-                except ValidTimeStampError:
-                    continue
-                timestamps.append(timestamp)
-                _line1 = line_splat[1]
-                if len(_line1) != 11:
-                    raise NotImplementedError("Unknown message number")
-                if (msg_no := _line1[-4:-1]) != "002":
-                    raise NotImplementedError(
-                        f"Message number {msg_no} not implemented"
-                    )
-                _line3 = lines[i + 2].strip().split(" ")
-                scale, range_resolution, n_gates, tilt_angle = [
-                    int(_line3[ind]) for ind in [0, 1, 2, 5]
-                ]
-                scales.append(scale)
-                tilt_angles.append(tilt_angle)
-                _line4 = lines[i + 3].strip()
-                profiles.append(_hex2backscatter(_line4, n_gates))
-
-        if len(timestamps) == 0:
-            raise ValidTimeStampError("No valid timestamps found in the file")
-        array = self._handle_large_values(np.array(profiles))
-        self.data["beta_raw"] = _scale_backscatter(array, scales)
-        if calibration_factor is None:
-            calibration_factor = 1.0
-        self.data["beta_raw"] *= calibration_factor
-        self.data["calibration_factor"] = calibration_factor
-        self.data["range"] = (
-            np.arange(n_gates) * range_resolution + range_resolution / 2
-        )
-        self.data["time"] = utils.datetime2decimal_hours(timestamps)
-        self.data["zenith_angle"] = np.median(tilt_angles)
-
-    def _check_timestamp(self, timestamp: datetime):
-        timestamp_components = str(timestamp.date()).split("-")
-        if self.expected_date is not None:
-            if timestamp_components != self.expected_date.split("-"):
-                raise ValidTimeStampError
-        if not self.date:
-            self.date = timestamp_components
-        assert timestamp_components == self.date
-
-    @staticmethod
-    def _handle_large_values(array: np.ndarray) -> np.ndarray:
-        ind = np.where(array > 524287)
-        if ind[0].size > 0:
-            array[ind] -= 1048576
-        return array
-
-
-def is_timestamp(timestamp: str) -> bool:
-    """Tests if the input string is formatted as -yyyy-mm-dd hh:mm:ss"""
-    reg_exp = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{6}")
-    if reg_exp.match(timestamp) is not None:
-        return True
-    return False
-
-
-def _hex2backscatter(data: str, n_gates: int):
-    """Converts hex string to backscatter values."""
-    n_chars = 5
-    return [
-        int(data[i : i + n_chars], 16) for i in range(0, n_gates * n_chars, n_chars)
-    ]
-
-
-def _scale_backscatter(data: np.ndarray, scales: list) -> np.ndarray:
-    """Scales backscatter values."""
-    unit_conversion_factor = 1e-8
-    scales_array = np.array(scales)
-    ind = np.where(scales_array != 100)
-    if ind[0].size > 0:
-        logging.info(f"{ind[0].size} profiles have not 100% scaling")
-        data[ind, :] *= scales_array[ind] / 100
-    return data * unit_conversion_factor