cloudnetpy 1.51.0__tar.gz → 1.52.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. {cloudnetpy-1.51.0/cloudnetpy.egg-info → cloudnetpy-1.52.0}/PKG-INFO +1 -1
  2. cloudnetpy-1.52.0/cloudnetpy/instruments/campbell_scientific.py +135 -0
  3. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/ceilometer.py +3 -0
  4. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/cl61d.py +5 -1
  5. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/hatpro.py +12 -3
  6. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/lufft.py +0 -3
  7. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/nc_lidar.py +1 -1
  8. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/pollyxt.py +6 -0
  9. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/plotting/plotting.py +10 -26
  10. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/products/mwr_multi.py +5 -2
  11. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/products/mwr_single.py +5 -2
  12. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/products/product_tools.py +13 -1
  13. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/version.py +1 -1
  14. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0/cloudnetpy.egg-info}/PKG-INFO +1 -1
  15. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy.egg-info/requires.txt +1 -1
  16. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/pyproject.toml +1 -1
  17. cloudnetpy-1.51.0/cloudnetpy/instruments/campbell_scientific.py +0 -110
  18. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/LICENSE +0 -0
  19. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/MANIFEST.in +0 -0
  20. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/README.md +0 -0
  21. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/__init__.py +0 -0
  22. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/categorize/__init__.py +0 -0
  23. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/categorize/atmos.py +0 -0
  24. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/categorize/atmos_utils.py +0 -0
  25. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/categorize/categorize.py +0 -0
  26. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/categorize/classify.py +0 -0
  27. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/categorize/containers.py +0 -0
  28. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/categorize/droplet.py +0 -0
  29. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/categorize/falling.py +0 -0
  30. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/categorize/freezing.py +0 -0
  31. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/categorize/insects.py +0 -0
  32. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/categorize/lidar.py +0 -0
  33. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/categorize/melting.py +0 -0
  34. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/categorize/model.py +0 -0
  35. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/categorize/mwr.py +0 -0
  36. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/categorize/radar.py +0 -0
  37. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/cloudnetarray.py +0 -0
  38. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/concat_lib.py +0 -0
  39. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/constants.py +0 -0
  40. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/datasource.py +0 -0
  41. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/exceptions.py +0 -0
  42. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/__init__.py +0 -0
  43. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/basta.py +0 -0
  44. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/ceilo.py +0 -0
  45. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/cloudnet_instrument.py +0 -0
  46. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/copernicus.py +0 -0
  47. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/disdrometer/__init__.py +0 -0
  48. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/disdrometer/common.py +0 -0
  49. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/disdrometer/parsivel.py +0 -0
  50. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/disdrometer/thies.py +0 -0
  51. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/galileo.py +0 -0
  52. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/instruments.py +0 -0
  53. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/mira.py +0 -0
  54. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/nc_radar.py +0 -0
  55. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/radiometrics.py +0 -0
  56. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/rpg.py +0 -0
  57. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/rpg_reader.py +0 -0
  58. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/vaisala.py +0 -0
  59. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/instruments/weather_station.py +0 -0
  60. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/metadata.py +0 -0
  61. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/__init__.py +0 -0
  62. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/file_handler.py +0 -0
  63. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/metadata.py +0 -0
  64. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/model_metadata.py +0 -0
  65. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/plotting/__init__.py +0 -0
  66. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/plotting/plot_meta.py +0 -0
  67. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/plotting/plot_tools.py +0 -0
  68. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/plotting/plotting.py +0 -0
  69. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/products/__init__.py +0 -0
  70. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/products/advance_methods.py +0 -0
  71. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/products/grid_methods.py +0 -0
  72. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/products/model_products.py +0 -0
  73. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/products/observation_products.py +0 -0
  74. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/products/product_resampling.py +0 -0
  75. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/products/tools.py +0 -0
  76. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/statistics/__init__.py +0 -0
  77. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/statistics/statistical_methods.py +0 -0
  78. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/__init__.py +0 -0
  79. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/e2e/__init__.py +0 -0
  80. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/e2e/conftest.py +0 -0
  81. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/e2e/process_cf/__init__.py +0 -0
  82. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +0 -0
  83. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +0 -0
  84. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/e2e/process_iwc/__init__.py +0 -0
  85. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +0 -0
  86. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +0 -0
  87. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/e2e/process_lwc/__init__.py +0 -0
  88. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +0 -0
  89. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +0 -0
  90. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/unit/__init__.py +0 -0
  91. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/unit/conftest.py +0 -0
  92. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +0 -0
  93. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +0 -0
  94. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/unit/test_model_products.py +0 -0
  95. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +0 -0
  96. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +0 -0
  97. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/unit/test_plotting.py +0 -0
  98. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +0 -0
  99. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/tests/unit/test_tools.py +0 -0
  100. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/model_evaluation/utils.py +0 -0
  101. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/output.py +0 -0
  102. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/plotting/__init__.py +0 -0
  103. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/plotting/plot_meta.py +0 -0
  104. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/products/__init__.py +0 -0
  105. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/products/classification.py +0 -0
  106. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/products/der.py +0 -0
  107. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/products/drizzle.py +0 -0
  108. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/products/drizzle_error.py +0 -0
  109. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/products/drizzle_tools.py +0 -0
  110. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/products/ier.py +0 -0
  111. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/products/iwc.py +0 -0
  112. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/products/lwc.py +0 -0
  113. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/products/mie_lu_tables.nc +0 -0
  114. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/py.typed +0 -0
  115. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy/utils.py +0 -0
  116. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy.egg-info/SOURCES.txt +0 -0
  117. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy.egg-info/dependency_links.txt +0 -0
  118. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/cloudnetpy.egg-info/top_level.txt +0 -0
  119. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/docs/source/conf.py +0 -0
  120. {cloudnetpy-1.51.0 → cloudnetpy-1.52.0}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cloudnetpy
3
- Version: 1.51.0
3
+ Version: 1.52.0
4
4
  Summary: Python package for Cloudnet processing
5
5
  Author: Simo Tukiainen
6
6
  License: MIT License
@@ -0,0 +1,135 @@
1
+ import binascii
2
+ import re
3
+ from datetime import datetime
4
+ from typing import NamedTuple
5
+
6
+ import numpy as np
7
+
8
+ from cloudnetpy import utils
9
+ from cloudnetpy.exceptions import InconsistentDataError, ValidTimeStampError
10
+ from cloudnetpy.instruments import instruments
11
+ from cloudnetpy.instruments.ceilometer import Ceilometer
12
+
13
+
14
+ class Cs135(Ceilometer):
15
+ def __init__(
16
+ self, full_path: str, site_meta: dict, expected_date: str | None = None
17
+ ):
18
+ super().__init__()
19
+ self.full_path = full_path
20
+ self.site_meta = site_meta
21
+ self.expected_date = expected_date
22
+ self.data = {}
23
+ self.metadata = {}
24
+ self.instrument = instruments.CS135
25
+
26
+ def read_ceilometer_file(self, calibration_factor: float | None = None) -> None:
27
+ with open(self.full_path, mode="rb") as f:
28
+ content = f.read()
29
+ timestamps = []
30
+ profiles = []
31
+ tilt_angles = []
32
+ range_resolutions = []
33
+
34
+ parts = re.split(rb"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{6}),", content)
35
+ for i in range(1, len(parts), 2):
36
+ timestamp = datetime.strptime(parts[i].decode(), "%Y-%m-%dT%H:%M:%S.%f")
37
+ try:
38
+ self._check_timestamp(timestamp)
39
+ except ValidTimeStampError:
40
+ continue
41
+ try:
42
+ message = _read_message(parts[i + 1])
43
+ except InvalidMessageError:
44
+ continue
45
+ profile = (message.data[:-2] * 1e-8) * (message.scale / 100)
46
+ timestamps.append(timestamp)
47
+ profiles.append(profile)
48
+ tilt_angles.append(message.tilt_angle)
49
+ range_resolutions.append(message.range_resolution)
50
+
51
+ if len(timestamps) == 0:
52
+ raise ValidTimeStampError("No valid timestamps found in the file")
53
+ range_resolution = range_resolutions[0]
54
+ n_gates = len(profiles[0])
55
+ if any(res != range_resolution for res in range_resolutions):
56
+ raise InconsistentDataError("Inconsistent range resolution")
57
+ if any(len(profile) != n_gates for profile in profiles):
58
+ raise InconsistentDataError("Inconsistent number of gates")
59
+
60
+ self.data["beta_raw"] = np.array(profiles)
61
+ if calibration_factor is None:
62
+ calibration_factor = 1.0
63
+ self.data["beta_raw"] *= calibration_factor
64
+ self.data["calibration_factor"] = calibration_factor
65
+ self.data["range"] = (
66
+ np.arange(n_gates) * range_resolution + range_resolution / 2
67
+ )
68
+ self.data["time"] = utils.datetime2decimal_hours(timestamps)
69
+ self.data["zenith_angle"] = np.median(tilt_angles)
70
+
71
+ def _check_timestamp(self, timestamp: datetime):
72
+ timestamp_components = str(timestamp.date()).split("-")
73
+ if self.expected_date is not None:
74
+ if timestamp_components != self.expected_date.split("-"):
75
+ raise ValidTimeStampError
76
+ if not self.date:
77
+ self.date = timestamp_components
78
+ assert timestamp_components == self.date
79
+
80
+
81
+ class Message(NamedTuple):
82
+ scale: int
83
+ range_resolution: int
84
+ laser_pulse_energy: int
85
+ laser_temperature: int
86
+ tilt_angle: int
87
+ background_light: int
88
+ pulse_quantity: int
89
+ sample_rate: int
90
+ data: np.ndarray
91
+
92
+
93
+ class InvalidMessageError(Exception):
94
+ pass
95
+
96
+
97
+ def _read_message(message: bytes) -> Message:
98
+ end_idx = message.index(3)
99
+ content = message[1 : end_idx + 1]
100
+ expected_checksum = int(message[end_idx + 1 : end_idx + 5], 16)
101
+ actual_checksum = _crc16(content)
102
+ if expected_checksum != actual_checksum:
103
+ raise InvalidMessageError(
104
+ "Invalid checksum: "
105
+ f"expected {expected_checksum:04x}, "
106
+ f"got {actual_checksum:04x}"
107
+ )
108
+ lines = message.splitlines()
109
+ if len(lines[0]) != 11:
110
+ raise NotImplementedError("Unknown message format")
111
+ if (msg_no := lines[0][-4:-1]) != b"002":
112
+ raise NotImplementedError(f"Message number {msg_no.decode()} not implemented")
113
+ if len(lines) != 5:
114
+ raise InvalidMessageError("Invalid line count")
115
+ scale, res, n, energy, lt, ti, bl, pulse, rate, _sum = map(int, lines[2].split())
116
+ data = _read_backscatter(lines[3].strip(), n)
117
+ return Message(scale, res, energy, lt, ti, bl, pulse, rate, data)
118
+
119
+
120
+ def _read_backscatter(data: bytes, n_gates: int) -> np.ndarray:
121
+ """Read backscatter values from hex-encoded two's complement values."""
122
+ n_chars = 5
123
+ n_bits = n_chars * 4
124
+ limit = (1 << (n_bits - 1)) - 1
125
+ offset = 1 << n_bits
126
+ out = np.array(
127
+ [int(data[i : i + n_chars], 16) for i in range(0, n_gates * n_chars, n_chars)]
128
+ )
129
+ out[out > limit] -= offset
130
+ return out
131
+
132
+
133
+ def _crc16(data: bytes) -> int:
134
+ """Compute checksum similar to CRC-16-CCITT."""
135
+ return binascii.crc_hqx(data, 0xFFFF) ^ 0xFFFF
@@ -21,6 +21,8 @@ class NoiseParam:
21
21
  class Ceilometer:
22
22
  """Base class for all types of ceilometers and pollyxt."""
23
23
 
24
+ serial_number: str | None
25
+
24
26
  def __init__(self, noise_param: NoiseParam = NoiseParam()):
25
27
  self.noise_param = noise_param
26
28
  self.data: dict = {} # Need to contain 'beta_raw', 'range' and 'time'
@@ -29,6 +31,7 @@ class Ceilometer:
29
31
  self.site_meta: dict = {}
30
32
  self.date: list[str] = []
31
33
  self.instrument: Instrument | None = None
34
+ self.serial_number = None
32
35
 
33
36
  def calc_screened_product(
34
37
  self,
@@ -23,7 +23,8 @@ class Cl61d(NcLidar):
23
23
  """Reads data and metadata from concatenated Vaisala CL61d netCDF file."""
24
24
  with netCDF4.Dataset(self.file_name) as dataset:
25
25
  self.dataset = dataset
26
- self._fetch_zenith_angle("zenith", default=3.0)
26
+ self._fetch_attributes()
27
+ self._fetch_zenith_angle("tilt_angle", default=3.0)
27
28
  self._fetch_range(reference="lower")
28
29
  self._fetch_lidar_variables(calibration_factor)
29
30
  self._fetch_time_and_date()
@@ -41,3 +42,6 @@ class Cl61d(NcLidar):
41
42
  self.data["depolarisation"] = (
42
43
  self.dataset.variables["x_pol"][:] / self.dataset.variables["p_pol"][:]
43
44
  )
45
+
46
+ def _fetch_attributes(self):
47
+ self.serial_number = getattr(self.dataset, "instrument_serial_number", None)
@@ -36,7 +36,7 @@ def hatpro2l1c(
36
36
  Args:
37
37
  mwr_dir: Folder containing one day of HATPRO files.
38
38
  output_file: Output file name.
39
- site_meta: Dictionary containing information about the site
39
+ site_meta: Dictionary containing information about the site and instrument
40
40
  uuid: Set specific UUID for the file.
41
41
  date: Expected date in the input files.
42
42
 
@@ -44,7 +44,16 @@ def hatpro2l1c(
44
44
  UUID of the generated file.
45
45
  """
46
46
 
47
- hatpro_raw = mwrpy.lev1_to_nc(site_meta["coeffs_dir"], "1C01", mwr_dir)
47
+ coeff_files = site_meta.get("coefficientFiles", None)
48
+
49
+ hatpro_raw = mwrpy.lev1_to_nc(
50
+ "1C01",
51
+ mwr_dir,
52
+ output_file=output_file,
53
+ coeff_files=coeff_files,
54
+ instrument_config=site_meta,
55
+ )
56
+
48
57
  hatpro = HatproL1c(hatpro_raw, site_meta)
49
58
 
50
59
  timestamps = hatpro.data["time"][:]
@@ -75,7 +84,7 @@ def hatpro2l1c(
75
84
  nc.cloudnet_file_type = "mwr-l1c"
76
85
  nc.title = nc.title.replace("radiometer", "radiometer Level 1c")
77
86
  nc.mwrpy_version = mwrpy_version
78
- nc.mwrpy_coefficients = site_meta["coeffs_dir"]
87
+ nc.mwrpy_coefficients = ", ".join(site_meta["coefficientLinks"])
79
88
 
80
89
  return uuid
81
90
 
@@ -12,8 +12,6 @@ from cloudnetpy.instruments.nc_lidar import NcLidar
12
12
  class LufftCeilo(NcLidar):
13
13
  """Class for Lufft chm15k ceilometer."""
14
14
 
15
- serial_number: str | None
16
-
17
15
  def __init__(
18
16
  self, file_name: str, site_meta: dict, expected_date: str | None = None
19
17
  ):
@@ -21,7 +19,6 @@ class LufftCeilo(NcLidar):
21
19
  self.file_name = file_name
22
20
  self.site_meta = site_meta
23
21
  self.expected_date = expected_date
24
- self.serial_number = None
25
22
 
26
23
  def read_ceilometer_file(self, calibration_factor: float | None = None) -> None:
27
24
  """Reads data and metadata from Jenoptik netCDF file."""
@@ -30,7 +30,7 @@ class NcLidar(Ceilometer):
30
30
  def _fetch_zenith_angle(self, key: str, default: float = 3.0) -> None:
31
31
  assert self.dataset is not None
32
32
  if key in self.dataset.variables:
33
- zenith_angle = self.dataset.variables[key][:]
33
+ zenith_angle = np.median(self.dataset.variables[key][:])
34
34
  else:
35
35
  zenith_angle = float(default)
36
36
  logging.warning(f"No zenith angle found, assuming {zenith_angle} degrees")
@@ -103,6 +103,7 @@ class PollyXt(Ceilometer):
103
103
  raise InconsistentDataError(
104
104
  "Inconsistent number of pollyxt bsc / depol files"
105
105
  )
106
+ self._fetch_attributes(bsc_files[0])
106
107
  self.data["range"] = _read_array_from_multiple_files(
107
108
  bsc_files, depol_files, "height"
108
109
  )
@@ -157,6 +158,11 @@ class PollyXt(Ceilometer):
157
158
  return channel
158
159
  raise ValidTimeStampError("No functional pollyXT backscatter channels found")
159
160
 
161
+ def _fetch_attributes(self, file: str) -> None:
162
+ with netCDF4.Dataset(file, "r") as nc:
163
+ if hasattr(nc, "source"):
164
+ self.serial_number = nc.source.lower()
165
+
160
166
 
161
167
  def _read_array_from_multiple_files(files1: list, files2: list, key) -> np.ndarray:
162
168
  array: np.ndarray = np.array([])
@@ -483,8 +483,7 @@ def _plot_disdrometer(ax, data: ndarray, time: ndarray, name: str, unit: str):
483
483
 
484
484
  def _plot_hatpro(ax, data: dict, full_path: str):
485
485
  tb = _pointing_filter(full_path, data["tb"])
486
- time = _pointing_filter(full_path, data["time"])
487
- ax.plot(time, tb, color="royalblue", linestyle="-", linewidth=1)
486
+ ax.plot(data["time"], tb, color="royalblue", linestyle="-", linewidth=1)
488
487
  set_ax(
489
488
  ax,
490
489
  max_y=np.max(tb) + 0.5,
@@ -493,36 +492,21 @@ def _plot_hatpro(ax, data: dict, full_path: str):
493
492
  )
494
493
 
495
494
 
496
- def _elevation_filter(full_path: str, data_field: ndarray, ele_range: tuple) -> ndarray:
497
- """Filters data for specified range of elevation angles."""
498
- with netCDF4.Dataset(full_path) as nc:
499
- if "ele" in nc.variables:
500
- elevation = ptools.read_nc_fields(full_path, "ele")
501
- if data_field.ndim > 1:
502
- data_field = data_field[
503
- (elevation >= ele_range[0]) & (elevation <= ele_range[1]), :
504
- ]
505
- else:
506
- data_field = data_field[
507
- (elevation >= ele_range[0]) & (elevation <= ele_range[1])
508
- ]
509
- return data_field
510
-
511
-
512
495
  def _pointing_filter(
513
- full_path: str, data_field: ndarray, ele_range: tuple = (0, 91), status: int = 0
496
+ full_path: str, data: ndarray, zenith_limit=5, status: int = 0
514
497
  ) -> ndarray:
515
- """Filters data according to pointing flag."""
498
+ """Filters data according to pointing flag and zenith angle."""
516
499
  with netCDF4.Dataset(full_path) as nc:
517
500
  if "pointing_flag" in nc.variables:
518
501
  pointing = ptools.read_nc_fields(full_path, "pointing_flag")
519
- assert isinstance(pointing, ndarray)
520
- pointing_screened = _elevation_filter(full_path, pointing, ele_range)
521
- if data_field.ndim > 1:
522
- data_field = data_field[pointing_screened == status, :]
502
+ zenith_angle = ptools.read_nc_fields(full_path, "zenith_angle")
503
+ if data.ndim > 1:
504
+ data[np.abs(zenith_angle) > zenith_limit, :] = ma.masked
505
+ data[pointing != status, :] = ma.masked
523
506
  else:
524
- data_field = data_field[pointing_screened == status]
525
- return data_field
507
+ data[np.abs(zenith_angle) > zenith_limit] = ma.masked
508
+ data[pointing != status] = ma.masked
509
+ return data
526
510
 
527
511
 
528
512
  def _plot_weather_station(ax, data: ndarray, time: ndarray, name: str):
@@ -1,3 +1,4 @@
1
+ import tempfile
1
2
  from tempfile import NamedTemporaryFile
2
3
 
3
4
  import netCDF4
@@ -12,7 +13,6 @@ def generate_mwr_multi(
12
13
  mwr_l1c_file: str, output_file: str, uuid: str | None = None
13
14
  ) -> str:
14
15
  file_uuid = uuid if uuid is not None else utils.get_uuid()
15
- coeffs = product_tools.get_mwrpy_coeffs(mwr_l1c_file)
16
16
 
17
17
  with (
18
18
  NamedTemporaryFile() as temp_file,
@@ -20,16 +20,19 @@ def generate_mwr_multi(
20
20
  NamedTemporaryFile() as rel_hum_file,
21
21
  NamedTemporaryFile() as t_pot_file,
22
22
  NamedTemporaryFile() as eq_temp_file,
23
+ tempfile.TemporaryDirectory() as temp_dir,
23
24
  ):
25
+ coeffs = product_tools.get_read_mwrpy_coeffs(mwr_l1c_file, temp_dir)
26
+
24
27
  for prod, file in zip(
25
28
  ("2P02", "2P03", "2P04", "2P07", "2P08"),
26
29
  (temp_file, abs_hum_file, rel_hum_file, t_pot_file, eq_temp_file),
27
30
  ):
28
31
  lev2_to_nc(
29
- coeffs,
30
32
  prod,
31
33
  mwr_l1c_file,
32
34
  file.name,
35
+ coeff_files=coeffs,
33
36
  temp_file=temp_file.name if prod not in ("2P02", "2P03") else None,
34
37
  hum_file=abs_hum_file.name if prod not in ("2P02", "2P03") else None,
35
38
  )
@@ -1,3 +1,4 @@
1
+ import tempfile
1
2
  from tempfile import NamedTemporaryFile
2
3
 
3
4
  import netCDF4
@@ -12,19 +13,21 @@ def generate_mwr_single(
12
13
  mwr_l1c_file: str, output_file: str, uuid: str | None = None
13
14
  ) -> str:
14
15
  file_uuid = uuid if uuid is not None else utils.get_uuid()
15
- coeffs = product_tools.get_mwrpy_coeffs(mwr_l1c_file)
16
16
 
17
17
  with (
18
18
  NamedTemporaryFile() as lwp_file,
19
19
  NamedTemporaryFile() as iwv_file,
20
20
  NamedTemporaryFile() as t_prof_file,
21
21
  NamedTemporaryFile() as abs_hum_file,
22
+ tempfile.TemporaryDirectory() as temp_dir,
22
23
  ):
24
+ coeffs = product_tools.get_read_mwrpy_coeffs(mwr_l1c_file, temp_dir)
25
+
23
26
  for prod, file in zip(
24
27
  ("2I01", "2I02", "2P01", "2P03"),
25
28
  (lwp_file, iwv_file, t_prof_file, abs_hum_file),
26
29
  ):
27
- lev2_to_nc(coeffs, prod, mwr_l1c_file, file.name)
30
+ lev2_to_nc(prod, mwr_l1c_file, file.name, coeff_files=coeffs)
28
31
 
29
32
  with (
30
33
  netCDF4.Dataset(output_file, "w", format="NETCDF4_CLASSIC") as nc_output,
@@ -1,8 +1,10 @@
1
1
  """General helper classes and functions for all products."""
2
+ import os
2
3
  from collections import namedtuple
3
4
 
4
5
  import netCDF4
5
6
  import numpy as np
7
+ import requests
6
8
  from numpy import ma
7
9
 
8
10
  from cloudnetpy import constants, utils
@@ -276,4 +278,14 @@ def get_temperature(categorize_file: str) -> np.ndarray:
276
278
 
277
279
  def get_mwrpy_coeffs(nc_file: str) -> str:
278
280
  with netCDF4.Dataset(nc_file) as nc:
279
- return nc.mwrpy_coefficients
281
+ return nc.mwrpy_coefficients.split(", ")
282
+
283
+
284
+ def get_read_mwrpy_coeffs(mwr_l1c_file, folder: str) -> list:
285
+ coeffs = []
286
+ for link in get_mwrpy_coeffs(mwr_l1c_file):
287
+ full_path = os.path.join(folder, link.split("/")[-1])
288
+ with open(full_path, "wb") as f:
289
+ f.write(requests.get(link, timeout=10).content)
290
+ coeffs.append(full_path)
291
+ return coeffs
@@ -1,4 +1,4 @@
1
1
  MAJOR = 1
2
- MINOR = 51
2
+ MINOR = 52
3
3
  PATCH = 0
4
4
  __version__ = f"{MAJOR}.{MINOR}.{PATCH}"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cloudnetpy
3
- Version: 1.51.0
3
+ Version: 1.52.0
4
4
  Summary: Python package for Cloudnet processing
5
5
  Author: Simo Tukiainen
6
6
  License: MIT License
@@ -1,6 +1,6 @@
1
1
  cloudnetpy_qc>=1.10.2
2
2
  matplotlib
3
- mwrpy>=0.1.6
3
+ mwrpy>=0.4.1
4
4
  netCDF4
5
5
  requests
6
6
  rpgpy>=0.14.2
@@ -17,7 +17,7 @@ classifiers = [
17
17
  dependencies = [
18
18
  "cloudnetpy_qc>=1.10.2",
19
19
  "matplotlib",
20
- "mwrpy>=0.1.6",
20
+ "mwrpy>=0.4.1",
21
21
  "netCDF4",
22
22
  "requests",
23
23
  "rpgpy>=0.14.2",
@@ -1,110 +0,0 @@
1
- import logging
2
- import re
3
- from datetime import datetime
4
-
5
- import numpy as np
6
-
7
- from cloudnetpy import utils
8
- from cloudnetpy.exceptions import ValidTimeStampError
9
- from cloudnetpy.instruments import instruments
10
- from cloudnetpy.instruments.ceilometer import Ceilometer
11
-
12
-
13
- class Cs135(Ceilometer):
14
- def __init__(
15
- self, full_path: str, site_meta: dict, expected_date: str | None = None
16
- ):
17
- super().__init__()
18
- self.full_path = full_path
19
- self.site_meta = site_meta
20
- self.expected_date = expected_date
21
- self.data = {}
22
- self.metadata = {}
23
- self.instrument = instruments.CS135
24
-
25
- def read_ceilometer_file(self, calibration_factor: float | None = None) -> None:
26
- with open(self.full_path, mode="r", encoding="utf-8") as f:
27
- lines = f.readlines()
28
- timestamps, profiles, scales, tilt_angles = [], [], [], []
29
- range_resolution, n_gates = 0, 0
30
- for i, line in enumerate(lines):
31
- line_splat = line.strip().split(",")
32
- if is_timestamp(line_splat[0]):
33
- timestamp = datetime.strptime(line_splat[0], "%Y-%m-%dT%H:%M:%S.%f")
34
- try:
35
- self._check_timestamp(timestamp)
36
- except ValidTimeStampError:
37
- continue
38
- timestamps.append(timestamp)
39
- _line1 = line_splat[1]
40
- if len(_line1) != 11:
41
- raise NotImplementedError("Unknown message number")
42
- if (msg_no := _line1[-4:-1]) != "002":
43
- raise NotImplementedError(
44
- f"Message number {msg_no} not implemented"
45
- )
46
- _line3 = lines[i + 2].strip().split(" ")
47
- scale, range_resolution, n_gates, tilt_angle = [
48
- int(_line3[ind]) for ind in [0, 1, 2, 5]
49
- ]
50
- scales.append(scale)
51
- tilt_angles.append(tilt_angle)
52
- _line4 = lines[i + 3].strip()
53
- profiles.append(_hex2backscatter(_line4, n_gates))
54
-
55
- if len(timestamps) == 0:
56
- raise ValidTimeStampError("No valid timestamps found in the file")
57
- array = self._handle_large_values(np.array(profiles))
58
- self.data["beta_raw"] = _scale_backscatter(array, scales)
59
- if calibration_factor is None:
60
- calibration_factor = 1.0
61
- self.data["beta_raw"] *= calibration_factor
62
- self.data["calibration_factor"] = calibration_factor
63
- self.data["range"] = (
64
- np.arange(n_gates) * range_resolution + range_resolution / 2
65
- )
66
- self.data["time"] = utils.datetime2decimal_hours(timestamps)
67
- self.data["zenith_angle"] = np.median(tilt_angles)
68
-
69
- def _check_timestamp(self, timestamp: datetime):
70
- timestamp_components = str(timestamp.date()).split("-")
71
- if self.expected_date is not None:
72
- if timestamp_components != self.expected_date.split("-"):
73
- raise ValidTimeStampError
74
- if not self.date:
75
- self.date = timestamp_components
76
- assert timestamp_components == self.date
77
-
78
- @staticmethod
79
- def _handle_large_values(array: np.ndarray) -> np.ndarray:
80
- ind = np.where(array > 524287)
81
- if ind[0].size > 0:
82
- array[ind] -= 1048576
83
- return array
84
-
85
-
86
- def is_timestamp(timestamp: str) -> bool:
87
- """Tests if the input string is formatted as -yyyy-mm-dd hh:mm:ss"""
88
- reg_exp = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{6}")
89
- if reg_exp.match(timestamp) is not None:
90
- return True
91
- return False
92
-
93
-
94
- def _hex2backscatter(data: str, n_gates: int):
95
- """Converts hex string to backscatter values."""
96
- n_chars = 5
97
- return [
98
- int(data[i : i + n_chars], 16) for i in range(0, n_gates * n_chars, n_chars)
99
- ]
100
-
101
-
102
- def _scale_backscatter(data: np.ndarray, scales: list) -> np.ndarray:
103
- """Scales backscatter values."""
104
- unit_conversion_factor = 1e-8
105
- scales_array = np.array(scales)
106
- ind = np.where(scales_array != 100)
107
- if ind[0].size > 0:
108
- logging.info(f"{ind[0].size} profiles have not 100% scaling")
109
- data[ind, :] *= scales_array[ind] / 100
110
- return data * unit_conversion_factor
File without changes
File without changes
File without changes
File without changes