disdrodb 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. disdrodb/__init__.py +68 -34
  2. disdrodb/_config.py +5 -4
  3. disdrodb/_version.py +16 -3
  4. disdrodb/accessor/__init__.py +20 -0
  5. disdrodb/accessor/methods.py +125 -0
  6. disdrodb/api/checks.py +177 -24
  7. disdrodb/api/configs.py +3 -3
  8. disdrodb/api/info.py +13 -13
  9. disdrodb/api/io.py +281 -22
  10. disdrodb/api/path.py +184 -195
  11. disdrodb/api/search.py +18 -9
  12. disdrodb/cli/disdrodb_create_summary.py +103 -0
  13. disdrodb/cli/disdrodb_create_summary_station.py +91 -0
  14. disdrodb/cli/disdrodb_run_l0.py +1 -1
  15. disdrodb/cli/disdrodb_run_l0_station.py +1 -1
  16. disdrodb/cli/disdrodb_run_l0a_station.py +1 -1
  17. disdrodb/cli/disdrodb_run_l0b.py +1 -1
  18. disdrodb/cli/disdrodb_run_l0b_station.py +3 -3
  19. disdrodb/cli/disdrodb_run_l0c.py +1 -1
  20. disdrodb/cli/disdrodb_run_l0c_station.py +3 -3
  21. disdrodb/cli/disdrodb_run_l1_station.py +2 -2
  22. disdrodb/cli/disdrodb_run_l2e_station.py +2 -2
  23. disdrodb/cli/disdrodb_run_l2m_station.py +2 -2
  24. disdrodb/configs.py +149 -4
  25. disdrodb/constants.py +61 -0
  26. disdrodb/data_transfer/download_data.py +127 -11
  27. disdrodb/etc/configs/attributes.yaml +339 -0
  28. disdrodb/etc/configs/encodings.yaml +473 -0
  29. disdrodb/etc/products/L1/global.yaml +13 -0
  30. disdrodb/etc/products/L2E/10MIN.yaml +12 -0
  31. disdrodb/etc/products/L2E/1MIN.yaml +1 -0
  32. disdrodb/etc/products/L2E/global.yaml +22 -0
  33. disdrodb/etc/products/L2M/10MIN.yaml +12 -0
  34. disdrodb/etc/products/L2M/GAMMA_ML.yaml +8 -0
  35. disdrodb/etc/products/L2M/NGAMMA_GS_LOG_ND_MAE.yaml +6 -0
  36. disdrodb/etc/products/L2M/NGAMMA_GS_ND_MAE.yaml +6 -0
  37. disdrodb/etc/products/L2M/NGAMMA_GS_Z_MAE.yaml +6 -0
  38. disdrodb/etc/products/L2M/global.yaml +26 -0
  39. disdrodb/issue/writer.py +2 -0
  40. disdrodb/l0/__init__.py +13 -0
  41. disdrodb/l0/configs/LPM/l0b_cf_attrs.yml +4 -4
  42. disdrodb/l0/configs/PARSIVEL/l0b_cf_attrs.yml +1 -1
  43. disdrodb/l0/configs/PARSIVEL/l0b_encodings.yml +3 -3
  44. disdrodb/l0/configs/PARSIVEL/raw_data_format.yml +1 -1
  45. disdrodb/l0/configs/PARSIVEL2/l0b_cf_attrs.yml +5 -5
  46. disdrodb/l0/configs/PARSIVEL2/l0b_encodings.yml +3 -3
  47. disdrodb/l0/configs/PARSIVEL2/raw_data_format.yml +1 -1
  48. disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +4 -4
  49. disdrodb/l0/configs/PWS100/raw_data_format.yml +1 -1
  50. disdrodb/l0/l0a_processing.py +37 -32
  51. disdrodb/l0/l0b_nc_processing.py +118 -8
  52. disdrodb/l0/l0b_processing.py +30 -65
  53. disdrodb/l0/l0c_processing.py +369 -259
  54. disdrodb/l0/readers/LPM/ARM/ARM_LPM.py +7 -0
  55. disdrodb/l0/readers/LPM/NETHERLANDS/DELFT_LPM_NC.py +66 -0
  56. disdrodb/l0/readers/LPM/SLOVENIA/{CRNI_VRH.py → UL.py} +3 -0
  57. disdrodb/l0/readers/LPM/SWITZERLAND/INNERERIZ_LPM.py +195 -0
  58. disdrodb/l0/readers/PARSIVEL/GPM/PIERS.py +0 -2
  59. disdrodb/l0/readers/PARSIVEL/JAPAN/JMA.py +4 -1
  60. disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +1 -1
  61. disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +1 -1
  62. disdrodb/l0/readers/PARSIVEL2/ARM/ARM_PARSIVEL2.py +4 -0
  63. disdrodb/l0/readers/PARSIVEL2/BELGIUM/ILVO.py +168 -0
  64. disdrodb/l0/readers/PARSIVEL2/CANADA/UQAM_NC.py +69 -0
  65. disdrodb/l0/readers/PARSIVEL2/DENMARK/DTU.py +165 -0
  66. disdrodb/l0/readers/PARSIVEL2/FINLAND/FMI_PARSIVEL2.py +69 -0
  67. disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +255 -134
  68. disdrodb/l0/readers/PARSIVEL2/FRANCE/OSUG.py +525 -0
  69. disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +1 -1
  70. disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +9 -7
  71. disdrodb/l0/readers/PARSIVEL2/KIT/BURKINA_FASO.py +1 -1
  72. disdrodb/l0/readers/PARSIVEL2/KIT/TEAMX.py +123 -0
  73. disdrodb/l0/readers/PARSIVEL2/{NETHERLANDS/DELFT.py → MPI/BCO_PARSIVEL2.py} +41 -71
  74. disdrodb/l0/readers/PARSIVEL2/MPI/BOWTIE.py +220 -0
  75. disdrodb/l0/readers/PARSIVEL2/NASA/APU.py +120 -0
  76. disdrodb/l0/readers/PARSIVEL2/NASA/LPVEX.py +109 -0
  77. disdrodb/l0/readers/PARSIVEL2/NCAR/FARM_PARSIVEL2.py +1 -0
  78. disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +1 -1
  79. disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_MIPS.py +126 -0
  80. disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_PIPS.py +165 -0
  81. disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +1 -1
  82. disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +20 -12
  83. disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT_NC.py +5 -0
  84. disdrodb/l0/readers/PARSIVEL2/SPAIN/CENER.py +144 -0
  85. disdrodb/l0/readers/PARSIVEL2/SPAIN/CR1000DL.py +201 -0
  86. disdrodb/l0/readers/PARSIVEL2/SPAIN/LIAISE.py +137 -0
  87. disdrodb/l0/readers/PARSIVEL2/USA/C3WE.py +146 -0
  88. disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +105 -99
  89. disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100_SIRTA.py +151 -0
  90. disdrodb/l1/__init__.py +5 -0
  91. disdrodb/l1/fall_velocity.py +46 -0
  92. disdrodb/l1/filters.py +34 -20
  93. disdrodb/l1/processing.py +46 -45
  94. disdrodb/l1/resampling.py +77 -66
  95. disdrodb/l1_env/routines.py +18 -3
  96. disdrodb/l2/__init__.py +7 -0
  97. disdrodb/l2/empirical_dsd.py +58 -10
  98. disdrodb/l2/processing.py +268 -117
  99. disdrodb/metadata/checks.py +132 -125
  100. disdrodb/metadata/standards.py +3 -1
  101. disdrodb/psd/fitting.py +631 -345
  102. disdrodb/psd/models.py +9 -6
  103. disdrodb/routines/__init__.py +54 -0
  104. disdrodb/{l0/routines.py → routines/l0.py} +316 -355
  105. disdrodb/{l1/routines.py → routines/l1.py} +76 -116
  106. disdrodb/routines/l2.py +1019 -0
  107. disdrodb/{routines.py → routines/wrappers.py} +98 -10
  108. disdrodb/scattering/__init__.py +16 -4
  109. disdrodb/scattering/axis_ratio.py +61 -37
  110. disdrodb/scattering/permittivity.py +504 -0
  111. disdrodb/scattering/routines.py +746 -184
  112. disdrodb/summary/__init__.py +17 -0
  113. disdrodb/summary/routines.py +4196 -0
  114. disdrodb/utils/archiving.py +434 -0
  115. disdrodb/utils/attrs.py +68 -125
  116. disdrodb/utils/cli.py +5 -5
  117. disdrodb/utils/compression.py +30 -1
  118. disdrodb/utils/dask.py +121 -9
  119. disdrodb/utils/dataframe.py +61 -7
  120. disdrodb/utils/decorators.py +31 -0
  121. disdrodb/utils/directories.py +35 -15
  122. disdrodb/utils/encoding.py +37 -19
  123. disdrodb/{l2 → utils}/event.py +15 -173
  124. disdrodb/utils/logger.py +14 -7
  125. disdrodb/utils/manipulations.py +81 -0
  126. disdrodb/utils/routines.py +166 -0
  127. disdrodb/utils/subsetting.py +214 -0
  128. disdrodb/utils/time.py +35 -177
  129. disdrodb/utils/writer.py +20 -7
  130. disdrodb/utils/xarray.py +5 -4
  131. disdrodb/viz/__init__.py +13 -0
  132. disdrodb/viz/plots.py +398 -0
  133. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/METADATA +4 -3
  134. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/RECORD +139 -98
  135. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/entry_points.txt +2 -0
  136. disdrodb/l1/encoding_attrs.py +0 -642
  137. disdrodb/l2/processing_options.py +0 -213
  138. disdrodb/l2/routines.py +0 -868
  139. /disdrodb/l0/readers/PARSIVEL/SLOVENIA/{UL_FGG.py → UL.py} +0 -0
  140. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/WHEEL +0 -0
  141. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/licenses/LICENSE +0 -0
  142. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,109 @@
1
+ #!/usr/bin/env python3
2
+ # -----------------------------------------------------------------------------.
3
+ # Copyright (c) 2021-2023 DISDRODB developers
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify
6
+ # it under the terms of the GNU General Public License as published by
7
+ # the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+ # -----------------------------------------------------------------------------.
18
+ """This reader allows to read raw data from NASA APU stations.
19
+
20
+ The reader allows to read raw APU data from the following NASA campaigns:
21
+
22
+ - HYMEX
23
+ - IFLOODS
24
+ - IPHEX
25
+ - OLYMPEX
26
+ - ICEPOP
27
+ - IMPACTS
28
+ - GCPEX
29
+ - WFF
30
+
31
+ """
32
+
33
+ import pandas as pd
34
+
35
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
36
+ from disdrodb.l0.l0a_processing import read_raw_text_file
37
+
38
+
39
+ @is_documented_by(reader_generic_docstring)
40
+ def reader(
41
+ filepath,
42
+ logger=None,
43
+ ):
44
+ """Reader."""
45
+ ##------------------------------------------------------------------------.
46
+ #### Define column names
47
+ column_names = ["time", "TO_BE_SPLITTED"]
48
+
49
+ ##------------------------------------------------------------------------.
50
+ #### Define reader options
51
+ reader_kwargs = {}
52
+ # - Define delimiter
53
+ reader_kwargs["delimiter"] = ";"
54
+ # - Skip first row as columns names
55
+ reader_kwargs["header"] = None
56
+ reader_kwargs["skiprows"] = 0
57
+ # - Skip file with encoding errors
58
+ reader_kwargs["encoding_errors"] = "ignore"
59
+ # - Avoid first column to become df index !!!
60
+ reader_kwargs["index_col"] = False
61
+ # - Define behaviour when encountering bad lines
62
+ reader_kwargs["on_bad_lines"] = "skip"
63
+ # - Define reader engine
64
+ # - C engine is faster
65
+ # - Python engine is more feature-complete
66
+ reader_kwargs["engine"] = "python"
67
+ # - Define on-the-fly decompression of on-disk data
68
+ # - Available: gzip, bz2, zip
69
+ reader_kwargs["compression"] = "infer"
70
+ # - Strings to recognize as NA/NaN and replace with standard NA flags
71
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
72
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
73
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
74
+ reader_kwargs["na_values"] = ["na", "", "error", "NA", "-.-"]
75
+
76
+ ##------------------------------------------------------------------------.
77
+ #### Read the data
78
+ df = read_raw_text_file(
79
+ filepath=filepath,
80
+ column_names=column_names,
81
+ reader_kwargs=reader_kwargs,
82
+ logger=logger,
83
+ )
84
+
85
+ ##------------------------------------------------------------------------.
86
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
87
+ # Convert time column to datetime
88
+ df_time = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
89
+
90
+ # Split the 'TO_BE_SPLITTED' column
91
+ df = df["TO_BE_SPLITTED"].str.split(",", n=3, expand=True)
92
+
93
+ # Assign column names
94
+ names = [
95
+ "station_name",
96
+ "unknown",
97
+ "unknown2",
98
+ "raw_drop_number",
99
+ ]
100
+ df.columns = names
101
+
102
+ # Add the time column
103
+ df["time"] = df_time
104
+
105
+ # Drop columns not agreeing with DISDRODB L0 standards
106
+ df = df.drop(columns=["station_name", "unknown", "unknown2"])
107
+
108
+ # Return the dataframe adhering to DISDRODB L0 standards
109
+ return df
@@ -128,6 +128,7 @@ def reader(
128
128
  columns_to_drop = [
129
129
  "station_name",
130
130
  "station_number",
131
+ "sensor_serial_number",
131
132
  "firmware_iop",
132
133
  "firmware_dsp",
133
134
  "TO_SPLIT",
@@ -108,7 +108,7 @@ def reader(
108
108
  # --> "" generates an array of zeros in L0B processing
109
109
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>ZERO</SPECTRUM>", "")
110
110
 
111
- # Remove <SPECTRUM> and </SPECTRUM>" acronyms from the raw_drop_number field
111
+ # Remove <SPECTRUM> and </SPECTRUM> prefix and suffix from the raw_drop_number field
112
112
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
113
113
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
114
114
 
@@ -0,0 +1,126 @@
1
+ #!/usr/bin/env python3
2
+ # -----------------------------------------------------------------------------.
3
+ # Copyright (c) 2021-2023 DISDRODB developers
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify
6
+ # it under the terms of the GNU General Public License as published by
7
+ # the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+ # -----------------------------------------------------------------------------.
18
+ """Reader for the PERILS 2022 and 2023 MIPS and RADAPS files."""
19
+ import pandas as pd
20
+
21
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
22
+ from disdrodb.l0.l0a_processing import read_raw_text_file
23
+
24
+
25
+ @is_documented_by(reader_generic_docstring)
26
+ def reader(
27
+ filepath,
28
+ logger=None,
29
+ ):
30
+ """Reader."""
31
+ ##------------------------------------------------------------------------.
32
+ #### Define column names
33
+ column_names = ["TO_PARSE"]
34
+
35
+ ##------------------------------------------------------------------------.
36
+ #### Define reader options
37
+ reader_kwargs = {}
38
+ # - Define delimiter
39
+ reader_kwargs["delimiter"] = "\\n"
40
+ # - Avoid first column to become df index !!!
41
+ reader_kwargs["index_col"] = False
42
+
43
+ # - Define behaviour when encountering bad lines
44
+ reader_kwargs["on_bad_lines"] = "skip"
45
+
46
+ # Skip first row as columns names
47
+ reader_kwargs["header"] = None
48
+
49
+ # - Define encoding
50
+ reader_kwargs["encoding"] = "ISO-8859-1"
51
+
52
+ # - Define reader engine
53
+ # - C engine is faster
54
+ # - Python engine is more feature-complete
55
+ reader_kwargs["engine"] = "python"
56
+
57
+ # - Define on-the-fly decompression of on-disk data
58
+ # - Available: gzip, bz2, zip
59
+ reader_kwargs["compression"] = "infer"
60
+
61
+ # - Strings to recognize as NA/NaN and replace with standard NA flags
62
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
63
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
64
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
65
+ reader_kwargs["na_values"] = ["na", "", "error"]
66
+
67
+ ##------------------------------------------------------------------------.
68
+ #### Read the data
69
+ df = read_raw_text_file(
70
+ filepath=filepath,
71
+ column_names=column_names,
72
+ reader_kwargs=reader_kwargs,
73
+ logger=logger,
74
+ )
75
+
76
+ ##------------------------------------------------------------------------.
77
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
78
+ # Split and assign integrated variables names
79
+ df = df["TO_PARSE"].str.split(",", expand=True, n=6)
80
+ names = [
81
+ "year",
82
+ "doy",
83
+ "time",
84
+ "sensor_temperature",
85
+ "number_particles",
86
+ "rainfall_rate_32bit",
87
+ "TO_SPLIT",
88
+ ]
89
+ df.columns = names
90
+
91
+ # Derive raw drop arrays
92
+ def split_string(s):
93
+ vals = [v.strip() for v in s.split(",")]
94
+ c1 = ", ".join(vals[:32])
95
+ c2 = ", ".join(vals[32:64])
96
+ # c3 = ", ".join(vals[64:])
97
+ series = pd.Series(
98
+ {
99
+ "raw_drop_concentration": c1,
100
+ "raw_drop_average_velocity": c2,
101
+ # "raw_drop_number": c3,
102
+ },
103
+ )
104
+ return series
105
+
106
+ splitted_string = df["TO_SPLIT"].apply(split_string)
107
+ df["raw_drop_concentration"] = splitted_string["raw_drop_concentration"]
108
+ df["raw_drop_average_velocity"] = splitted_string["raw_drop_average_velocity"]
109
+
110
+ # Define datetime time column
111
+ df["year"] = df["year"].str.replace(".0", "")
112
+ df["doy"] = df["doy"].str.replace(".0", "")
113
+ df["time"] = df["time"].str.replace(".0", "")
114
+ df["time"] = df["year"].astype(str) + "-" + df["doy"].astype(str) + " " + df["time"].astype(str)
115
+ df["time"] = pd.to_datetime(df["time"], format="%Y-%j %H%M%S", errors="coerce")
116
+
117
+ # Drop columns not agreeing with DISDRODB L0 standards
118
+ columns_to_drop = [
119
+ "doy",
120
+ "year",
121
+ "TO_SPLIT",
122
+ ]
123
+ df = df.drop(columns=columns_to_drop)
124
+
125
+ # Return the dataframe adhering to DISDRODB L0 standards
126
+ return df
@@ -0,0 +1,165 @@
1
+ #!/usr/bin/env python3
2
+ # -----------------------------------------------------------------------------.
3
+ # Copyright (c) 2021-2023 DISDRODB developers
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify
6
+ # it under the terms of the GNU General Public License as published by
7
+ # the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+ # -----------------------------------------------------------------------------.
18
+ """Reader for the PERILS 2022 and 2023 PIPS files."""
19
+ import pandas as pd
20
+
21
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
22
+ from disdrodb.l0.l0a_processing import read_raw_text_file
23
+
24
+
25
+ @is_documented_by(reader_generic_docstring)
26
+ def reader(
27
+ filepath,
28
+ logger=None,
29
+ ):
30
+ """Reader."""
31
+ ##------------------------------------------------------------------------.
32
+ #### Define column names
33
+ column_names = [
34
+ "logger_time",
35
+ "record_number",
36
+ "battery_voltage",
37
+ "logger_temperature",
38
+ "wind_direction",
39
+ "wind_speed",
40
+ "wind_flag",
41
+ "fast_temperature",
42
+ "slow_temperature",
43
+ "relative_humidity",
44
+ "pressure",
45
+ "compass_direction",
46
+ "gps_time",
47
+ "gps_status",
48
+ "gps_latitude", # DD.DM, DM = decimal minutes/100, DD = degrees
49
+ "gps_latitude_hemisphere", # N/S
50
+ "gps_longitude",
51
+ "gps_longitude_hemisphere", # W/E
52
+ "gps_speed",
53
+ "gps_direction",
54
+ "gps_date",
55
+ "gps_magnetic_version",
56
+ "gps_altitude",
57
+ "relative_wind_direction",
58
+ "dew_point_temperature",
59
+ "rh",
60
+ "disdrometer_data",
61
+ ]
62
+
63
+ ##------------------------------------------------------------------------.
64
+ #### Define reader options
65
+ reader_kwargs = {}
66
+
67
+ # - Define delimiter
68
+ reader_kwargs["delimiter"] = ","
69
+
70
+ # - Avoid first column to become df index !!!
71
+ reader_kwargs["index_col"] = False
72
+
73
+ # - Define behaviour when encountering bad lines
74
+ reader_kwargs["on_bad_lines"] = "skip"
75
+
76
+ # - Define encoding
77
+ reader_kwargs["encoding"] = "ISO-8859-1"
78
+
79
+ # - Define reader engine
80
+ # - C engine is faster
81
+ # - Python engine is more feature-complete
82
+ reader_kwargs["engine"] = "python"
83
+
84
+ # - Define on-the-fly decompression of on-disk data
85
+ # - Available: gzip, bz2, zip
86
+ reader_kwargs["compression"] = "infer"
87
+
88
+ # - Strings to recognize as NA/NaN and replace with standard NA flags
89
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
90
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
91
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
92
+ reader_kwargs["na_values"] = ["na", "", "error"]
93
+
94
+ # Skip first row as columns names
95
+ reader_kwargs["header"] = None
96
+
97
+ ##------------------------------------------------------------------------.
98
+ #### Read the data
99
+ df = read_raw_text_file(
100
+ filepath=filepath,
101
+ column_names=column_names,
102
+ reader_kwargs=reader_kwargs,
103
+ logger=logger,
104
+ )
105
+
106
+ ##------------------------------------------------------------------------.
107
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
108
+ # Drop timesteps without disdrometer data
109
+ df = df[~df["disdrometer_data"].isna()]
110
+
111
+ # Retrieve disdrometer data
112
+ df_data = df["disdrometer_data"].str.split(";", expand=True, n=11)
113
+
114
+ # Assign column names
115
+ column_names = [
116
+ "serial_number",
117
+ "rainfall_rate_32bit",
118
+ "rainfall_accumulated_32bit",
119
+ "reflectivity_32bit",
120
+ "sample_interval",
121
+ "laser_amplitude",
122
+ "number_particles",
123
+ "sensor_temperature",
124
+ "sensor_battery_voltage",
125
+ "sensor_time", # Note: logger_time is currently used !
126
+ "sensor_date",
127
+ "raw_drop_number",
128
+ ]
129
+ df_data.columns = column_names
130
+
131
+ # Add weather information
132
+ df_data["air_temperature"] = df["fast_temperature"]
133
+ df_data["relative_humidity"] = df["relative_humidity"]
134
+ df_data["wind_direction"] = df["wind_direction"]
135
+ df_data["wind_speed"] = df["wind_speed"]
136
+
137
+ # df_data["dew_point"] = df["dew_point"]
138
+ # df_data["air_pressure"] = df["pressure"]
139
+
140
+ # Retrieve time
141
+ df_time = pd.to_datetime(df["logger_time"], errors="coerce")
142
+
143
+ # Retrieve coordinates information
144
+ # --> Latitude in degrees_north
145
+ # --> Longitude in degrees_east
146
+ # df_lat_sign = df["gps_latitude_hemisphere"].str.replace("N", "1").str.replace("S", "-1")
147
+ # df_lon_sign = df["gps_longitude_hemisphere"].str.replace("E", "1").str.replace("W", "-1")
148
+ # df_lat_sign = df_lat_sign.astype(float)
149
+ # df_lon_sign = df_lon_sign.astype(float)
150
+ # df_lon = df["gps_longitude"].astype(float)
151
+ # df_lat = df["gps_latitude"].astype(float)
152
+ # df_lon = df_lon * df_lon_sign
153
+ # df_lat = df_lat * df_lat_sign
154
+
155
+ # Create dataframe
156
+ df = df_data
157
+ df["time"] = df_time
158
+ # df["latitude"] = df_lat
159
+ # df["longitude"] = df_lon
160
+
161
+ # Drop columns not agreeing with DISDRODB L0 standards
162
+ df = df.drop(columns=["serial_number", "sensor_time", "sensor_date", "serial_number"])
163
+
164
+ # Return the dataframe adhering to DISDRODB L0 standards
165
+ return df
@@ -102,7 +102,7 @@ def reader(
102
102
  df["raw_drop_number"] = df["raw_drop_number"].str.strip()
103
103
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>ZERO</SPECTRUM>", "")
104
104
 
105
- # Remove <SPECTRUM> and </SPECTRUM>" acronyms from the raw_drop_number field
105
+ # Remove <SPECTRUM> and </SPECTRUM> prefix and suffix from the raw_drop_number field
106
106
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
107
107
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
108
108
 
@@ -15,6 +15,9 @@
15
15
  # You should have received a copy of the GNU General Public License
16
16
  # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
17
  # -----------------------------------------------------------------------------.
18
+ """Reader for the PERILS 2022 and 2023 PIPS files."""
19
+ import os
20
+
18
21
  import pandas as pd
19
22
 
20
23
  from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
@@ -62,6 +65,7 @@ def reader(
62
65
  ##------------------------------------------------------------------------.
63
66
  #### Define reader options
64
67
  reader_kwargs = {}
68
+
65
69
  # - Define delimiter
66
70
  reader_kwargs["delimiter"] = ","
67
71
 
@@ -91,6 +95,8 @@ def reader(
91
95
 
92
96
  # Skip first row as columns names
93
97
  reader_kwargs["header"] = None
98
+ if "FMCW" in os.path.basename(filepath):
99
+ reader_kwargs["skiprows"] = 1
94
100
 
95
101
  ##------------------------------------------------------------------------.
96
102
  #### Read the data
@@ -135,24 +141,26 @@ def reader(
135
141
  # df_data["dew_point"] = df["dew_point"]
136
142
  # df_data["air_pressure"] = df["pressure"]
137
143
 
138
- # Retrieve time and coordinates information
144
+ # Retrieve time
145
+ df_time = pd.to_datetime(df["logger_time"], errors="coerce")
146
+
147
+ # Retrieve coordinates information
139
148
  # --> Latitude in degrees_north
140
149
  # --> Longitude in degrees_east
141
- df_time = pd.to_datetime(df["logger_time"], errors="coerce")
142
- df_lat_sign = df["gps_latitude_hemisphere"].str.replace("N", "1").str.replace("S", "-1")
143
- df_lon_sign = df["gps_longitude_hemisphere"].str.replace("E", "1").str.replace("W", "-1")
144
- df_lat_sign = df_lat_sign.astype(float)
145
- df_lon_sign = df_lon_sign.astype(float)
146
- df_lon = df["gps_longitude"].astype(float)
147
- df_lat = df["gps_latitude"].astype(float)
148
- df_lon = df_lon * df_lon_sign
149
- df_lat = df_lat * df_lat_sign
150
+ # df_lat_sign = df["gps_latitude_hemisphere"].str.replace("N", "1").str.replace("S", "-1")
151
+ # df_lon_sign = df["gps_longitude_hemisphere"].str.replace("E", "1").str.replace("W", "-1")
152
+ # df_lat_sign = df_lat_sign.astype(float)
153
+ # df_lon_sign = df_lon_sign.astype(float)
154
+ # df_lon = df["gps_longitude"].astype(float)
155
+ # df_lat = df["gps_latitude"].astype(float)
156
+ # df_lon = df_lon * df_lon_sign
157
+ # df_lat = df_lat * df_lat_sign
150
158
 
151
159
  # Create dataframe
152
160
  df = df_data
153
161
  df["time"] = df_time
154
- df["latitude"] = df_lat
155
- df["longitude"] = df_lon
162
+ # df["latitude"] = df_lat
163
+ # df["longitude"] = df_lon
156
164
 
157
165
  # Drop columns not agreeing with DISDRODB L0 standards
158
166
  df = df.drop(columns=["serial_number", "sensor_time", "sensor_date", "serial_number"])
@@ -35,6 +35,8 @@ def reader(
35
35
  #### Adapt the dataframe to adhere to DISDRODB L0 standards
36
36
  # Add time coordinate
37
37
  ds["time"] = ds["time_as_string"].astype("M8[s]")
38
+ ds["time"].attrs.pop("comment", None)
39
+ ds["time"].attrs.pop("units", None)
38
40
  ds = ds.set_coords("time")
39
41
 
40
42
  # Define dictionary mapping dataset variables to select and rename
@@ -63,5 +65,8 @@ def reader(
63
65
  # Rename dataset variables and columns and infill missing variables
64
66
  ds = standardize_raw_dataset(ds=ds, dict_names=dict_names, sensor_name="PARSIVEL2")
65
67
 
68
+ # Replace NaN flags
69
+ ds["raw_drop_number"] = ds["raw_drop_number"].where(ds["raw_drop_number"] < 65437) # dummy flag 65437.0
70
+
66
71
  # Return the dataset adhering to DISDRODB L0B standards
67
72
  return ds
@@ -0,0 +1,144 @@
1
+ #!/usr/bin/env python3
2
+ # -----------------------------------------------------------------------------.
3
+ # Copyright (c) 2021-2023 DISDRODB developers
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify
6
+ # it under the terms of the GNU General Public License as published by
7
+ # the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+ # -----------------------------------------------------------------------------.
18
+ import pandas as pd
19
+
20
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
21
+ from disdrodb.l0.l0a_processing import read_raw_text_file
22
+
23
+
24
+ @is_documented_by(reader_generic_docstring)
25
+ def reader(
26
+ filepath,
27
+ logger=None,
28
+ ):
29
+ """Reader."""
30
+ ##------------------------------------------------------------------------.
31
+ #### Define column names
32
+ column_names = ["TO_PARSE"]
33
+
34
+ ##------------------------------------------------------------------------.
35
+ #### Define reader options
36
+ reader_kwargs = {}
37
+ # - Define delimiter
38
+ reader_kwargs["delimiter"] = "\\n"
39
+ # - Skip first row as columns names
40
+ # - Define encoding
41
+ reader_kwargs["encoding"] = "latin" # "ISO-8859-1"
42
+ # - Avoid first column to become df index !!!
43
+ reader_kwargs["index_col"] = False
44
+ # - Define behaviour when encountering bad lines
45
+ reader_kwargs["on_bad_lines"] = "skip"
46
+ # - Define reader engine
47
+ # - C engine is faster
48
+ # - Python engine is more feature-complete
49
+ reader_kwargs["engine"] = "python"
50
+ # - Define on-the-fly decompression of on-disk data
51
+ # - Available: gzip, bz2, zip
52
+ reader_kwargs["compression"] = "infer"
53
+ # - Strings to recognize as NA/NaN and replace with standard NA flags
54
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
55
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
56
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
57
+ reader_kwargs["na_values"] = ["na", "", "error"]
58
+
59
+ ##------------------------------------------------------------------------.
60
+ #### Read the data
61
+ df = read_raw_text_file(
62
+ filepath=filepath,
63
+ column_names=column_names,
64
+ reader_kwargs=reader_kwargs,
65
+ logger=logger,
66
+ )
67
+
68
+ ##------------------------------------------------------------------------.
69
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
70
+ # Define time
71
+ df = df["TO_PARSE"].str.split(",", n=2, expand=True)
72
+ df.columns = ["date", "time", "TO_PARSE"]
73
+ datetime_str = df["date"] + " " + df["time"]
74
+ df["time"] = pd.to_datetime(datetime_str, format="%d.%m.%Y %H:%M:%S", errors="coerce")
75
+
76
+ # Identify rows with integral variables
77
+ df_vars = df[df["TO_PARSE"].str.len() == 94]
78
+
79
+ # Split and assign column names
80
+ df_data = df_vars["TO_PARSE"].str.split(",", expand=True)
81
+ var_names = [
82
+ "rainfall_rate_32bit",
83
+ "rainfall_accumulated_32bit",
84
+ "weather_code_synop_4680",
85
+ "weather_code_synop_4677",
86
+ "reflectivity_32bit",
87
+ "mor_visibility",
88
+ "laser_amplitude",
89
+ "number_particles",
90
+ "sensor_temperature",
91
+ "sensor_heating_current",
92
+ "sensor_battery_voltage",
93
+ "sensor_status",
94
+ "sensor_serial_number",
95
+ "sensor_temperature_receiver",
96
+ "sensor_temperature_trasmitter",
97
+ "snowfall_rate",
98
+ "rain_kinetic_energy",
99
+ ]
100
+ df_data.columns = var_names
101
+ df_data["time"] = df_vars["time"]
102
+ df_data = df_data.drop(columns="sensor_serial_number")
103
+
104
+ # Initialize empty arrays
105
+ # --> 0 values array produced in L0B
106
+ df_data["raw_drop_concentration"] = ""
107
+ df_data["raw_drop_average_velocity"] = ""
108
+ df_data["raw_drop_number"] = ""
109
+
110
+ # Identify raw spectrum
111
+ df_raw_spectrum = df[df["TO_PARSE"].str.len() == 4545]
112
+
113
+ # Derive raw drop arrays
114
+ def split_string(s):
115
+ vals = [v.strip() for v in s.split(",")]
116
+ c1 = ",".join(vals[:32])
117
+ c2 = ",".join(vals[32:64])
118
+ c3 = ",".join(vals[64].replace("r", "").split("/"))
119
+ series = pd.Series(
120
+ {
121
+ "raw_drop_concentration": c1,
122
+ "raw_drop_average_velocity": c2,
123
+ "raw_drop_number": c3,
124
+ },
125
+ )
126
+ return series
127
+
128
+ splitted_string = df_raw_spectrum["TO_PARSE"].apply(split_string)
129
+ df_raw_spectrum["raw_drop_concentration"] = splitted_string["raw_drop_concentration"]
130
+ df_raw_spectrum["raw_drop_average_velocity"] = splitted_string["raw_drop_average_velocity"]
131
+ df_raw_spectrum["raw_drop_number"] = splitted_string["raw_drop_number"]
132
+ df_raw_spectrum = df_raw_spectrum.drop(columns=["date", "TO_PARSE"])
133
+
134
+ # Add raw array
135
+ df = df_data.set_index("time")
136
+ df_raw_spectrum = df_raw_spectrum.set_index("time")
137
+
138
+ df.update(df_raw_spectrum)
139
+
140
+ # Set back time as column
141
+ df = df.reset_index()
142
+
143
+ # Return the dataframe adhering to DISDRODB L0 standards
144
+ return df