disdrodb 0.0.21__py3-none-any.whl → 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (264) hide show
  1. disdrodb/__init__.py +132 -15
  2. disdrodb/_config.py +4 -2
  3. disdrodb/_version.py +9 -4
  4. disdrodb/api/checks.py +264 -237
  5. disdrodb/api/configs.py +4 -8
  6. disdrodb/api/create_directories.py +235 -290
  7. disdrodb/api/info.py +217 -26
  8. disdrodb/api/io.py +295 -269
  9. disdrodb/api/path.py +597 -173
  10. disdrodb/api/search.py +486 -0
  11. disdrodb/{metadata/scripts → cli}/disdrodb_check_metadata_archive.py +12 -7
  12. disdrodb/{utils/pandas.py → cli/disdrodb_data_archive_directory.py} +9 -18
  13. disdrodb/cli/disdrodb_download_archive.py +86 -0
  14. disdrodb/cli/disdrodb_download_metadata_archive.py +53 -0
  15. disdrodb/cli/disdrodb_download_station.py +84 -0
  16. disdrodb/{api/scripts → cli}/disdrodb_initialize_station.py +22 -10
  17. disdrodb/cli/disdrodb_metadata_archive_directory.py +32 -0
  18. disdrodb/{data_transfer/scripts/disdrodb_download_station.py → cli/disdrodb_open_data_archive.py} +22 -22
  19. disdrodb/cli/disdrodb_open_logs_directory.py +69 -0
  20. disdrodb/{data_transfer/scripts/disdrodb_upload_station.py → cli/disdrodb_open_metadata_archive.py} +22 -24
  21. disdrodb/cli/disdrodb_open_metadata_directory.py +71 -0
  22. disdrodb/cli/disdrodb_open_product_directory.py +74 -0
  23. disdrodb/cli/disdrodb_open_readers_directory.py +32 -0
  24. disdrodb/{l0/scripts → cli}/disdrodb_run_l0.py +38 -31
  25. disdrodb/{l0/scripts → cli}/disdrodb_run_l0_station.py +32 -30
  26. disdrodb/{l0/scripts → cli}/disdrodb_run_l0a.py +30 -21
  27. disdrodb/{l0/scripts → cli}/disdrodb_run_l0a_station.py +24 -33
  28. disdrodb/{l0/scripts → cli}/disdrodb_run_l0b.py +30 -21
  29. disdrodb/{l0/scripts → cli}/disdrodb_run_l0b_station.py +25 -34
  30. disdrodb/cli/disdrodb_run_l0c.py +130 -0
  31. disdrodb/cli/disdrodb_run_l0c_station.py +129 -0
  32. disdrodb/cli/disdrodb_run_l1.py +122 -0
  33. disdrodb/cli/disdrodb_run_l1_station.py +121 -0
  34. disdrodb/cli/disdrodb_run_l2e.py +122 -0
  35. disdrodb/cli/disdrodb_run_l2e_station.py +122 -0
  36. disdrodb/cli/disdrodb_run_l2m.py +122 -0
  37. disdrodb/cli/disdrodb_run_l2m_station.py +122 -0
  38. disdrodb/cli/disdrodb_upload_archive.py +105 -0
  39. disdrodb/cli/disdrodb_upload_station.py +98 -0
  40. disdrodb/configs.py +90 -25
  41. disdrodb/data_transfer/__init__.py +22 -0
  42. disdrodb/data_transfer/download_data.py +87 -90
  43. disdrodb/data_transfer/upload_data.py +64 -37
  44. disdrodb/data_transfer/zenodo.py +15 -18
  45. disdrodb/docs.py +1 -1
  46. disdrodb/issue/__init__.py +17 -4
  47. disdrodb/issue/checks.py +10 -23
  48. disdrodb/issue/reader.py +9 -12
  49. disdrodb/issue/writer.py +14 -17
  50. disdrodb/l0/__init__.py +17 -26
  51. disdrodb/l0/check_configs.py +35 -23
  52. disdrodb/l0/check_standards.py +32 -42
  53. disdrodb/l0/configs/{Thies_LPM → LPM}/bins_diameter.yml +44 -44
  54. disdrodb/l0/configs/{Thies_LPM → LPM}/bins_velocity.yml +40 -40
  55. disdrodb/l0/configs/LPM/l0a_encodings.yml +80 -0
  56. disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_cf_attrs.yml +62 -59
  57. disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_encodings.yml +9 -9
  58. disdrodb/l0/configs/{Thies_LPM → LPM}/raw_data_format.yml +245 -245
  59. disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_diameter.yml +66 -66
  60. disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_velocity.yml +64 -64
  61. disdrodb/l0/configs/PARSIVEL/l0a_encodings.yml +32 -0
  62. disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_cf_attrs.yml +22 -20
  63. disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_encodings.yml +17 -17
  64. disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/raw_data_format.yml +77 -77
  65. disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_diameter.yml +64 -64
  66. disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_velocity.yml +64 -64
  67. disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +39 -0
  68. disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_cf_attrs.yml +24 -22
  69. disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_encodings.yml +20 -20
  70. disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/raw_data_format.yml +98 -98
  71. disdrodb/l0/configs/{RD_80 → RD80}/bins_diameter.yml +40 -40
  72. disdrodb/l0/configs/RD80/l0a_encodings.yml +16 -0
  73. disdrodb/l0/configs/{RD_80 → RD80}/l0b_cf_attrs.yml +3 -3
  74. disdrodb/l0/configs/RD80/l0b_encodings.yml +135 -0
  75. disdrodb/l0/configs/{RD_80 → RD80}/raw_data_format.yml +48 -48
  76. disdrodb/l0/l0_reader.py +216 -340
  77. disdrodb/l0/l0a_processing.py +237 -208
  78. disdrodb/l0/l0b_nc_processing.py +227 -80
  79. disdrodb/l0/l0b_processing.py +93 -173
  80. disdrodb/l0/l0c_processing.py +627 -0
  81. disdrodb/l0/readers/{ARM → LPM/ARM}/ARM_LPM.py +36 -58
  82. disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +226 -0
  83. disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +185 -0
  84. disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +183 -0
  85. disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +179 -0
  86. disdrodb/l0/readers/{UK → LPM/UK}/DIVEN.py +14 -35
  87. disdrodb/l0/readers/PARSIVEL/AUSTRALIA/MELBOURNE_2007_PARSIVEL.py +157 -0
  88. disdrodb/l0/readers/PARSIVEL/CHINA/CHONGQING.py +113 -0
  89. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/ARCTIC_2021.py +40 -57
  90. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/COMMON_2011.py +37 -54
  91. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/DAVOS_2009_2011.py +34 -51
  92. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_2009.py +34 -51
  93. disdrodb/l0/readers/{EPFL/PARADISO_2014.py → PARSIVEL/EPFL/EPFL_ROOF_2008.py} +38 -50
  94. disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2010.py +105 -0
  95. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2011.py +34 -51
  96. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2012.py +33 -51
  97. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GENEPI_2007.py +25 -44
  98. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007.py +25 -44
  99. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007_2.py +25 -44
  100. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HPICONET_2010.py +34 -51
  101. disdrodb/l0/readers/{EPFL/EPFL_ROOF_2010.py → PARSIVEL/EPFL/HYMEX_LTE_SOP2.py} +37 -50
  102. disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP3.py +111 -0
  103. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HYMEX_LTE_SOP4.py +36 -54
  104. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2018.py +34 -52
  105. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2019.py +38 -56
  106. disdrodb/l0/readers/PARSIVEL/EPFL/PARADISO_2014.py +105 -0
  107. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PARSIVEL_2007.py +27 -45
  108. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PLATO_2019.py +24 -44
  109. disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019.py +140 -0
  110. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RACLETS_2019_WJF.py +41 -59
  111. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RIETHOLZBACH_2011.py +34 -51
  112. disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2017.py +117 -0
  113. disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2019.py +137 -0
  114. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/UNIL_2022.py +42 -55
  115. disdrodb/l0/readers/PARSIVEL/GPM/IFLOODS.py +104 -0
  116. disdrodb/l0/readers/{GPM → PARSIVEL/GPM}/LPVEX.py +29 -48
  117. disdrodb/l0/readers/PARSIVEL/GPM/MC3E.py +184 -0
  118. disdrodb/l0/readers/PARSIVEL/NCAR/CCOPE_2015.py +113 -0
  119. disdrodb/l0/readers/{NCAR/VORTEX_SE_2016_P1.py → PARSIVEL/NCAR/OWLES_MIPS.py} +46 -72
  120. disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +125 -0
  121. disdrodb/l0/readers/{NCAR/OWLES_MIPS.py → PARSIVEL/NCAR/PLOWS_MIPS.py} +45 -64
  122. disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +114 -0
  123. disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +176 -0
  124. disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +183 -0
  125. disdrodb/l0/readers/{ARM/ARM_LD.py → PARSIVEL2/ARM/ARM_PARSIVEL2.py} +27 -50
  126. disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +163 -0
  127. disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +163 -0
  128. disdrodb/l0/readers/{DENMARK → PARSIVEL2/DENMARK}/EROSION_nc.py +14 -35
  129. disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +119 -0
  130. disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +104 -0
  131. disdrodb/l0/readers/PARSIVEL2/GPM/NSSTC.py +176 -0
  132. disdrodb/l0/readers/PARSIVEL2/ITALY/GID_PARSIVEL2.py +32 -0
  133. disdrodb/l0/readers/PARSIVEL2/MEXICO/OH_IIUNAM_nc.py +56 -0
  134. disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +120 -0
  135. disdrodb/l0/readers/{NCAR → PARSIVEL2/NCAR}/PECAN_MIPS.py +45 -64
  136. disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +181 -0
  137. disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +160 -0
  138. disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +160 -0
  139. disdrodb/l0/readers/{NCAR/PLOWS_MIPS.py → PARSIVEL2/NCAR/VORTEX_SE_2016_P1.py} +49 -66
  140. disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +118 -0
  141. disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +152 -0
  142. disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT.py +166 -0
  143. disdrodb/l0/readers/{NCAR/RELAMPAGO_RD80.py → RD80/BRAZIL/CHUVA_RD80.py} +36 -60
  144. disdrodb/l0/readers/{BRAZIL → RD80/BRAZIL}/GOAMAZON_RD80.py +36 -55
  145. disdrodb/l0/readers/{NCAR → RD80/NCAR}/CINDY_2011_RD80.py +35 -54
  146. disdrodb/l0/readers/{BRAZIL/CHUVA_RD80.py → RD80/NCAR/RELAMPAGO_RD80.py} +40 -54
  147. disdrodb/l0/readers/template_reader_raw_netcdf_data.py +62 -0
  148. disdrodb/l0/readers/{reader_template.py → template_reader_raw_text_data.py} +20 -44
  149. disdrodb/l0/routines.py +885 -581
  150. disdrodb/l0/standards.py +72 -236
  151. disdrodb/l0/template_tools.py +104 -109
  152. disdrodb/l1/__init__.py +17 -0
  153. disdrodb/l1/beard_model.py +716 -0
  154. disdrodb/l1/encoding_attrs.py +620 -0
  155. disdrodb/l1/fall_velocity.py +260 -0
  156. disdrodb/l1/filters.py +192 -0
  157. disdrodb/l1/processing.py +200 -0
  158. disdrodb/l1/resampling.py +236 -0
  159. disdrodb/l1/routines.py +357 -0
  160. disdrodb/l1_env/__init__.py +17 -0
  161. disdrodb/l1_env/routines.py +38 -0
  162. disdrodb/l2/__init__.py +17 -0
  163. disdrodb/l2/empirical_dsd.py +1735 -0
  164. disdrodb/l2/event.py +388 -0
  165. disdrodb/l2/processing.py +519 -0
  166. disdrodb/l2/processing_options.py +213 -0
  167. disdrodb/l2/routines.py +868 -0
  168. disdrodb/metadata/__init__.py +9 -2
  169. disdrodb/metadata/checks.py +165 -118
  170. disdrodb/metadata/download.py +81 -0
  171. disdrodb/metadata/geolocation.py +146 -0
  172. disdrodb/metadata/info.py +20 -13
  173. disdrodb/metadata/manipulation.py +1 -1
  174. disdrodb/metadata/reader.py +59 -8
  175. disdrodb/metadata/search.py +77 -144
  176. disdrodb/metadata/standards.py +7 -8
  177. disdrodb/metadata/writer.py +8 -14
  178. disdrodb/psd/__init__.py +38 -0
  179. disdrodb/psd/fitting.py +2146 -0
  180. disdrodb/psd/models.py +774 -0
  181. disdrodb/routines.py +1176 -0
  182. disdrodb/scattering/__init__.py +28 -0
  183. disdrodb/scattering/axis_ratio.py +344 -0
  184. disdrodb/scattering/routines.py +456 -0
  185. disdrodb/utils/__init__.py +17 -0
  186. disdrodb/utils/attrs.py +208 -0
  187. disdrodb/utils/cli.py +269 -0
  188. disdrodb/utils/compression.py +60 -42
  189. disdrodb/utils/dask.py +62 -0
  190. disdrodb/utils/decorators.py +110 -0
  191. disdrodb/utils/directories.py +107 -46
  192. disdrodb/utils/encoding.py +127 -0
  193. disdrodb/utils/list.py +29 -0
  194. disdrodb/utils/logger.py +168 -46
  195. disdrodb/utils/time.py +657 -0
  196. disdrodb/utils/warnings.py +30 -0
  197. disdrodb/utils/writer.py +57 -0
  198. disdrodb/utils/xarray.py +138 -47
  199. disdrodb/utils/yaml.py +0 -1
  200. disdrodb/viz/__init__.py +17 -0
  201. disdrodb/viz/plots.py +17 -0
  202. disdrodb-0.1.0.dist-info/METADATA +321 -0
  203. disdrodb-0.1.0.dist-info/RECORD +216 -0
  204. {disdrodb-0.0.21.dist-info → disdrodb-0.1.0.dist-info}/WHEEL +1 -1
  205. disdrodb-0.1.0.dist-info/entry_points.txt +30 -0
  206. disdrodb/data_transfer/scripts/disdrodb_download_archive.py +0 -53
  207. disdrodb/data_transfer/scripts/disdrodb_upload_archive.py +0 -57
  208. disdrodb/l0/configs/OTT_Parsivel/l0a_encodings.yml +0 -32
  209. disdrodb/l0/configs/OTT_Parsivel2/l0a_encodings.yml +0 -39
  210. disdrodb/l0/configs/RD_80/l0a_encodings.yml +0 -16
  211. disdrodb/l0/configs/RD_80/l0b_encodings.yml +0 -135
  212. disdrodb/l0/configs/Thies_LPM/l0a_encodings.yml +0 -80
  213. disdrodb/l0/io.py +0 -257
  214. disdrodb/l0/l0_processing.py +0 -1091
  215. disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_OTT.py +0 -178
  216. disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_THIES.py +0 -247
  217. disdrodb/l0/readers/BRAZIL/CHUVA_LPM.py +0 -204
  218. disdrodb/l0/readers/BRAZIL/CHUVA_OTT.py +0 -183
  219. disdrodb/l0/readers/BRAZIL/GOAMAZON_LPM.py +0 -204
  220. disdrodb/l0/readers/BRAZIL/GOAMAZON_OTT.py +0 -183
  221. disdrodb/l0/readers/CHINA/CHONGQING.py +0 -131
  222. disdrodb/l0/readers/EPFL/EPFL_ROOF_2008.py +0 -128
  223. disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP2.py +0 -127
  224. disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP3.py +0 -129
  225. disdrodb/l0/readers/EPFL/RACLETS_2019.py +0 -158
  226. disdrodb/l0/readers/EPFL/SAMOYLOV_2017.py +0 -136
  227. disdrodb/l0/readers/EPFL/SAMOYLOV_2019.py +0 -158
  228. disdrodb/l0/readers/FRANCE/SIRTA_OTT2.py +0 -138
  229. disdrodb/l0/readers/GPM/GCPEX.py +0 -123
  230. disdrodb/l0/readers/GPM/IFLOODS.py +0 -123
  231. disdrodb/l0/readers/GPM/MC3E.py +0 -123
  232. disdrodb/l0/readers/GPM/NSSTC.py +0 -164
  233. disdrodb/l0/readers/ITALY/GID.py +0 -199
  234. disdrodb/l0/readers/MEXICO/OH_IIUNAM_nc.py +0 -92
  235. disdrodb/l0/readers/NCAR/CCOPE_2015.py +0 -133
  236. disdrodb/l0/readers/NCAR/PECAN_FP3.py +0 -137
  237. disdrodb/l0/readers/NCAR/PECAN_MOBILE.py +0 -144
  238. disdrodb/l0/readers/NCAR/RELAMPAGO_OTT.py +0 -195
  239. disdrodb/l0/readers/NCAR/SNOWIE_PJ.py +0 -172
  240. disdrodb/l0/readers/NCAR/SNOWIE_SB.py +0 -179
  241. disdrodb/l0/readers/NCAR/VORTEX2_2009.py +0 -133
  242. disdrodb/l0/readers/NCAR/VORTEX2_2010.py +0 -188
  243. disdrodb/l0/readers/NCAR/VORTEX2_2010_UF.py +0 -191
  244. disdrodb/l0/readers/NCAR/VORTEX_SE_2016_P2.py +0 -135
  245. disdrodb/l0/readers/NCAR/VORTEX_SE_2016_PIPS.py +0 -170
  246. disdrodb/l0/readers/NETHERLANDS/DELFT.py +0 -187
  247. disdrodb/l0/readers/SPAIN/SBEGUERIA.py +0 -179
  248. disdrodb/l0/scripts/disdrodb_run_l0b_concat.py +0 -93
  249. disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py +0 -85
  250. disdrodb/utils/netcdf.py +0 -452
  251. disdrodb/utils/scripts.py +0 -102
  252. disdrodb-0.0.21.dist-info/AUTHORS.md +0 -18
  253. disdrodb-0.0.21.dist-info/METADATA +0 -186
  254. disdrodb-0.0.21.dist-info/RECORD +0 -168
  255. disdrodb-0.0.21.dist-info/entry_points.txt +0 -15
  256. /disdrodb/l0/configs/{RD_80 → RD80}/bins_velocity.yml +0 -0
  257. /disdrodb/l0/manuals/{Thies_LPM.pdf → LPM.pdf} +0 -0
  258. /disdrodb/l0/manuals/{ODM_470.pdf → ODM470.pdf} +0 -0
  259. /disdrodb/l0/manuals/{OTT_Parsivel.pdf → PARSIVEL.pdf} +0 -0
  260. /disdrodb/l0/manuals/{OTT_Parsivel2.pdf → PARSIVEL2.pdf} +0 -0
  261. /disdrodb/l0/manuals/{PWS_100.pdf → PWS100.pdf} +0 -0
  262. /disdrodb/l0/manuals/{RD_80.pdf → RD80.pdf} +0 -0
  263. {disdrodb-0.0.21.dist-info → disdrodb-0.1.0.dist-info/licenses}/LICENSE +0 -0
  264. {disdrodb-0.0.21.dist-info → disdrodb-0.1.0.dist-info}/top_level.txt +0 -0
@@ -15,23 +15,21 @@
15
15
  # You should have received a copy of the GNU General Public License
16
16
  # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
17
  # -----------------------------------------------------------------------------.
18
- from disdrodb.l0 import run_l0a
18
+ """Reader for UNIL 2022 campaign."""
19
+ import pandas as pd
20
+
19
21
  from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
22
+ from disdrodb.l0.l0a_processing import read_raw_text_file
20
23
 
21
24
 
22
25
  @is_documented_by(reader_generic_docstring)
23
26
  def reader(
24
- raw_dir,
25
- processed_dir,
26
- station_name,
27
- # Processing options
28
- force=False,
29
- verbose=False,
30
- parallel=False,
31
- debugging_mode=False,
27
+ filepath,
28
+ logger=None,
32
29
  ):
30
+ """Reader."""
33
31
  ##------------------------------------------------------------------------.
34
- #### - Define column names
32
+ #### Define column names
35
33
  column_names = [
36
34
  "id",
37
35
  "latitude",
@@ -59,7 +57,7 @@ def reader(
59
57
  ]
60
58
 
61
59
  ##------------------------------------------------------------------------.
62
- #### - Define reader options
60
+ #### Define reader options
63
61
  reader_kwargs = {}
64
62
  # - Define delimiter
65
63
  reader_kwargs["delimiter"] = ";"
@@ -75,55 +73,44 @@ def reader(
75
73
  # - Available: gzip, bz2, zip
76
74
  reader_kwargs["compression"] = "infer"
77
75
  # - Strings to recognize as NA/NaN and replace with standard NA flags
78
- # - Already included: ‘#N/A’, ‘#N/A N/A’, ‘#NA’, ‘-1.#IND’, ‘-1.#QNAN’,
79
- # ‘-NaN’, ‘-nan’, 1.#IND’, 1.#QNAN’, ‘<NA>’, N/A’,
80
- # NA’, NULL’, NaN’, n/a’, nan’, null
76
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
77
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
78
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
81
79
  reader_kwargs["na_values"] = ["na", "", "error"]
82
80
 
83
81
  ##------------------------------------------------------------------------.
84
- #### - Define dataframe sanitizer function for L0 processing
85
- def df_sanitizer_fun(df):
86
- # - Import pandas
87
- import pandas as pd
82
+ #### Read the data
83
+ df = read_raw_text_file(
84
+ filepath=filepath,
85
+ column_names=column_names,
86
+ reader_kwargs=reader_kwargs,
87
+ logger=logger,
88
+ )
88
89
 
89
- # - Convert time column to datetime with resolution in seconds
90
- df["time"] = pd.to_datetime(df["time"], format="%d-%m-%Y %H:%M:%S", errors="coerce")
90
+ ##------------------------------------------------------------------------.
91
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
92
+ # Convert time column to datetime with resolution in seconds
93
+ df["time"] = pd.to_datetime(df["time"], format="%d-%m-%Y %H:%M:%S", errors="coerce")
91
94
 
92
- # - Split TO_BE_SPLITTED columns
93
- df_splitted = df["TO_BE_SPLITTED"].str.split(",", expand=True, n=1)
94
- df_splitted.columns = ["datalogger_voltage", "rainfall_rate_32bit"]
95
- df["rainfall_rate_32bit"] = df_splitted["rainfall_rate_32bit"]
95
+ # Split TO_BE_SPLITTED columns
96
+ df_splitted = df["TO_BE_SPLITTED"].str.split(",", expand=True, n=1)
97
+ df_splitted.columns = ["datalogger_voltage", "rainfall_rate_32bit"]
98
+ df["rainfall_rate_32bit"] = df_splitted["rainfall_rate_32bit"]
96
99
 
97
- # - Drop columns not agreeing with DISDRODB L0 standards
98
- columns_to_drop = [
99
- "id",
100
- "TO_BE_SPLITTED",
101
- "datalogger_temperature",
102
- "datalogger_error",
103
- "latitude",
104
- "longitude",
105
- ]
106
- df = df.drop(columns=columns_to_drop)
107
- return df
100
+ # Remove rows with error in data reading
101
+ # - When datalogger error: rainfall_rate_32bit: Error in data reading!
102
+ df = df[df["rainfall_rate_32bit"] != "Error in data reading! 0"]
108
103
 
109
- ##------------------------------------------------------------------------.
110
- #### - Define glob pattern to search data files in raw_dir/data/<station_name>
111
- glob_patterns = "*.dat.gz"
104
+ # Drop columns not agreeing with DISDRODB L0 standards
105
+ columns_to_drop = [
106
+ "id",
107
+ "TO_BE_SPLITTED",
108
+ "datalogger_temperature",
109
+ "datalogger_error",
110
+ "latitude",
111
+ "longitude",
112
+ ]
113
+ df = df.drop(columns=columns_to_drop)
112
114
 
113
- ####----------------------------------------------------------------------.
114
- #### - Create L0A products
115
- run_l0a(
116
- raw_dir=raw_dir,
117
- processed_dir=processed_dir,
118
- station_name=station_name,
119
- # Custom arguments of the reader for L0A processing
120
- glob_patterns=glob_patterns,
121
- column_names=column_names,
122
- reader_kwargs=reader_kwargs,
123
- df_sanitizer_fun=df_sanitizer_fun,
124
- # Processing options
125
- force=force,
126
- verbose=verbose,
127
- parallel=parallel,
128
- debugging_mode=debugging_mode,
129
- )
115
+ # Return the dataframe adhering to DISDRODB L0 standards
116
+ return df
@@ -0,0 +1,104 @@
1
+ #!/usr/bin/env python3
2
+ # -----------------------------------------------------------------------------.
3
+ # Copyright (c) 2021-2023 DISDRODB developers
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify
6
+ # it under the terms of the GNU General Public License as published by
7
+ # the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+ # -----------------------------------------------------------------------------.
18
+ import pandas as pd
19
+
20
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
21
+ from disdrodb.l0.l0a_processing import read_raw_text_file
22
+
23
+
24
+ @is_documented_by(reader_generic_docstring)
25
+ def reader(
26
+ filepath,
27
+ logger=None,
28
+ ):
29
+ """Reader."""
30
+ ##------------------------------------------------------------------------.
31
+ #### Define column names
32
+ column_names = ["time", "TO_BE_SPLITTED"]
33
+
34
+ ##------------------------------------------------------------------------.
35
+ #### Define reader options
36
+ reader_kwargs = {}
37
+ # - Define delimiter
38
+ reader_kwargs["delimiter"] = ";"
39
+ # - Skip first row as columns names
40
+ reader_kwargs["header"] = None
41
+ # - Skip file with encoding errors
42
+ reader_kwargs["encoding_errors"] = "ignore"
43
+ # - Avoid first column to become df index !!!
44
+ reader_kwargs["index_col"] = False
45
+ # - Define behaviour when encountering bad lines
46
+ reader_kwargs["on_bad_lines"] = "skip"
47
+ # - Define reader engine
48
+ # - C engine is faster
49
+ # - Python engine is more feature-complete
50
+ reader_kwargs["engine"] = "python"
51
+ # - Define on-the-fly decompression of on-disk data
52
+ # - Available: gzip, bz2, zip
53
+ reader_kwargs["compression"] = "infer"
54
+ # - Strings to recognize as NA/NaN and replace with standard NA flags
55
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
56
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
57
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
58
+ reader_kwargs["na_values"] = ["na", "", "error", "NA", "-.-"]
59
+
60
+ ##------------------------------------------------------------------------.
61
+ #### Read the data
62
+ df = read_raw_text_file(
63
+ filepath=filepath,
64
+ column_names=column_names,
65
+ reader_kwargs=reader_kwargs,
66
+ logger=logger,
67
+ )
68
+
69
+ ##------------------------------------------------------------------------.
70
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
71
+ # Convert time column to datetime
72
+ df_time = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
73
+
74
+ # Split the 'TO_BE_SPLITTED' column
75
+ df = df["TO_BE_SPLITTED"].str.split(",", expand=True, n=9)
76
+
77
+ # Assign column names
78
+ columns_names = [
79
+ "station_name",
80
+ "sensor_status",
81
+ "sensor_temperature",
82
+ "number_particles",
83
+ "rainfall_rate_32bit",
84
+ "reflectivity_16bit",
85
+ "mor_visibility",
86
+ "weather_code_synop_4680",
87
+ "weather_code_synop_4677",
88
+ "raw_drop_number",
89
+ ]
90
+ df.columns = columns_names
91
+
92
+ # Add the time column
93
+ df["time"] = df_time
94
+
95
+ # Drop columns not agreeing with DISDRODB L0 standards
96
+ df = df.drop(columns=["station_name"])
97
+
98
+ # Drop rows with invalid values
99
+ # --> Ensure that weather_code_synop_4677 has length 2
100
+ # --> If a previous column is missing it will have 000
101
+ df = df[df["weather_code_synop_4677"].str.len() == 2]
102
+
103
+ # Return the dataframe adhering to DISDRODB L0 standards
104
+ return df
@@ -15,27 +15,24 @@
15
15
  # You should have received a copy of the GNU General Public License
16
16
  # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
17
  # -----------------------------------------------------------------------------.
18
- from disdrodb.l0 import run_l0a
18
+ import pandas as pd
19
+
19
20
  from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
21
+ from disdrodb.l0.l0a_processing import read_raw_text_file
20
22
 
21
23
 
22
24
  @is_documented_by(reader_generic_docstring)
23
25
  def reader(
24
- raw_dir,
25
- processed_dir,
26
- station_name,
27
- # Processing options
28
- force=False,
29
- verbose=False,
30
- parallel=False,
31
- debugging_mode=False,
26
+ filepath,
27
+ logger=None,
32
28
  ):
29
+ """Reader."""
33
30
  ##------------------------------------------------------------------------.
34
- #### - Define column names
31
+ #### Define column names
35
32
  column_names = ["time", "unknown", "raw_drop_number"]
36
33
 
37
34
  ##------------------------------------------------------------------------.
38
- #### - Define reader options
35
+ #### Define reader options
39
36
  reader_kwargs = {}
40
37
  # - Define delimiter
41
38
  reader_kwargs["delimiter"] = " "
@@ -59,46 +56,30 @@ def reader(
59
56
  # - Available: gzip, bz2, zip
60
57
  reader_kwargs["compression"] = "infer"
61
58
  # - Strings to recognize as NA/NaN and replace with standard NA flags
62
- # - Already included: ‘#N/A’, ‘#N/A N/A’, ‘#NA’, ‘-1.#IND’, ‘-1.#QNAN’,
63
- # ‘-NaN’, ‘-nan’, 1.#IND’, 1.#QNAN’, ‘<NA>’, N/A’,
64
- # NA’, NULL’, NaN’, n/a’, nan’, null
59
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
60
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
61
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
65
62
  reader_kwargs["na_values"] = ["na", "", "error", "-.-"]
66
63
 
67
64
  ##------------------------------------------------------------------------.
68
- #### - Define dataframe sanitizer function for L0 processing
69
- def df_sanitizer_fun(df):
70
- # - Import pandas
71
- import pandas as pd
72
-
73
- # - Convert time column to datetime
74
- try:
75
- df["time"] = pd.to_datetime(df["time"], format="%Y %m %d %H %M %S", errors="coerce")
76
- except ValueError:
77
- df["time"] = pd.to_datetime(df["time"], format="%Y-%m-%d %H:%M:%S", errors="coerce")
78
-
79
- # - Drop columns not agreeing with DISDRODB L0 standards
80
- df = df.drop(columns=["unknown"])
81
-
82
- return df
83
-
84
- ##------------------------------------------------------------------------.
85
- #### - Define glob pattern to search data files in raw_dir/data/<station_name>
86
- glob_patterns = "*.tar"
87
-
88
- ####----------------------------------------------------------------------.
89
- #### - Create L0A products
90
- run_l0a(
91
- raw_dir=raw_dir,
92
- processed_dir=processed_dir,
93
- station_name=station_name,
94
- # Custom arguments of the reader for L0A processing
95
- glob_patterns=glob_patterns,
65
+ #### Read the data
66
+ df = read_raw_text_file(
67
+ filepath=filepath,
96
68
  column_names=column_names,
97
69
  reader_kwargs=reader_kwargs,
98
- df_sanitizer_fun=df_sanitizer_fun,
99
- # Processing options
100
- force=force,
101
- verbose=verbose,
102
- parallel=parallel,
103
- debugging_mode=debugging_mode,
70
+ logger=logger,
104
71
  )
72
+
73
+ ##------------------------------------------------------------------------.
74
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
75
+ # Convert time column to datetime
76
+ try:
77
+ df["time"] = pd.to_datetime(df["time"], format="%Y %m %d %H %M %S", errors="coerce")
78
+ except ValueError:
79
+ df["time"] = pd.to_datetime(df["time"], format="%Y-%m-%d %H:%M:%S", errors="coerce")
80
+
81
+ # Drop columns not agreeing with DISDRODB L0 standards
82
+ df = df.drop(columns=["unknown"])
83
+
84
+ # Return the dataframe adhering to DISDRODB L0 standards
85
+ return df
@@ -0,0 +1,184 @@
1
+ #!/usr/bin/env python3
2
+ # -----------------------------------------------------------------------------.
3
+ # Copyright (c) 2021-2023 DISDRODB developers
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify
6
+ # it under the terms of the GNU General Public License as published by
7
+ # the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+ # -----------------------------------------------------------------------------.
18
+ import numpy as np
19
+ import pandas as pd
20
+
21
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
22
+ from disdrodb.l0.l0a_processing import read_raw_text_file
23
+
24
+
25
+ @is_documented_by(reader_generic_docstring)
26
+ def reader(
27
+ filepath,
28
+ logger=None,
29
+ ):
30
+ """Reader."""
31
+ ##------------------------------------------------------------------------.
32
+ #### Define column names
33
+ column_names = ["time", "TO_BE_SPLITTED"]
34
+
35
+ ##------------------------------------------------------------------------.
36
+ #### Define reader options
37
+ reader_kwargs = {}
38
+ # - Define delimiter
39
+ reader_kwargs["delimiter"] = ";"
40
+ # - Skip first row as columns names
41
+ reader_kwargs["header"] = None
42
+ # - Skip file with encoding errors
43
+ reader_kwargs["encoding_errors"] = "ignore"
44
+ # - Avoid first column to become df index
45
+ reader_kwargs["index_col"] = False
46
+ # - Define behaviour when encountering bad lines
47
+ reader_kwargs["on_bad_lines"] = "skip"
48
+ # - Define reader engine
49
+ # - C engine is faster
50
+ # - Python engine is more feature-complete
51
+ reader_kwargs["engine"] = "python"
52
+ # - Define on-the-fly decompression of on-disk data
53
+ # - Available: gzip, bz2, zip
54
+ reader_kwargs["compression"] = "infer"
55
+ # - Strings to recognize as NA/NaN and replace with standard NA flags
56
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
57
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
58
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
59
+ reader_kwargs["na_values"] = ["na", "", "error", "-.-"]
60
+
61
+ ##------------------------------------------------------------------------.
62
+ #### Read the data
63
+ df = read_raw_text_file(
64
+ filepath=filepath,
65
+ column_names=column_names,
66
+ reader_kwargs=reader_kwargs,
67
+ logger=logger,
68
+ )
69
+
70
+ ##------------------------------------------------------------------------.
71
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
72
+ # Convert 'time' column to datetime
73
+ df["time"] = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
74
+
75
+ # Count number of delimiters in the column to be parsed
76
+ # --> Some first rows are corrupted, so count the most frequent occurrence
77
+ possible_delimiters, counts = np.unique(df["TO_BE_SPLITTED"].str.count(","), return_counts=True)
78
+ n_delimiters = possible_delimiters[np.argmax(counts)]
79
+
80
+ # ---------------------------------------------------------
81
+ #### Case of 1031 delimiters
82
+ if n_delimiters == 1031: # first files
83
+ # Select valid rows
84
+ df = df.loc[df["TO_BE_SPLITTED"].str.count(",") == 1031]
85
+ # Get time column
86
+ df_time = df["time"]
87
+ # Split the 'TO_BE_SPLITTED' column
88
+ df = df["TO_BE_SPLITTED"].str.split(",", expand=True, n=7)
89
+ # Assign column names
90
+ column_names = [
91
+ "station_name",
92
+ "sensor_status",
93
+ "sensor_temperature",
94
+ "reflectivity_32bit",
95
+ "mor_visibility",
96
+ "weather_code_synop_4680",
97
+ "weather_code_synop_4677",
98
+ "raw_drop_number",
99
+ ]
100
+ df.columns = column_names
101
+ # Add time column
102
+ df["time"] = df_time
103
+ # Remove columns not in other files
104
+ df = df.drop(columns="reflectivity_32bit")
105
+ # Add missing columns and set NaN value
106
+ missing_columns = [
107
+ "number_particles",
108
+ "rainfall_rate_32bit",
109
+ "reflectivity_16bit",
110
+ ]
111
+ for column in missing_columns:
112
+ df[column] = "NaN"
113
+ # Drop columns not agreeing with DISDRODB L0 standards
114
+ df = df.drop(columns=["station_name"])
115
+ # Return the dataframe adhering to DISDRODB L0 standards
116
+ return df
117
+ # ---------------------------------------------------------
118
+ #### Case of 1032 delimiters
119
+ if n_delimiters == 1033: # (most of the files)
120
+ # Select valid rows
121
+ df = df.loc[df["TO_BE_SPLITTED"].str.count(",") == 1033]
122
+ # Get time column
123
+ df_time = df["time"]
124
+ # Split the column be parsed
125
+ df = df["TO_BE_SPLITTED"].str.split(",", expand=True, n=9)
126
+ # Assign column names
127
+ column_names = [
128
+ "station_name",
129
+ "sensor_status",
130
+ "sensor_temperature",
131
+ "number_particles",
132
+ "rainfall_rate_32bit",
133
+ "reflectivity_16bit",
134
+ "mor_visibility",
135
+ "weather_code_synop_4680",
136
+ "weather_code_synop_4677",
137
+ "raw_drop_number",
138
+ ]
139
+ df.columns = column_names
140
+ # Add time column
141
+ df["time"] = df_time
142
+ # Drop columns not agreeing with DISDRODB L0 standards
143
+ df = df.drop(columns=["station_name"])
144
+ # Return the dataframe adhering to DISDRODB L0 standards
145
+ return df
146
+
147
+ # ---------------------------------------------------------
148
+ #### Case of 1035 delimiters
149
+ if n_delimiters == 1035: # APU 17 first files
150
+ # Select valid rows
151
+ df = df.loc[df["TO_BE_SPLITTED"].str.count(",") == 1035]
152
+ # Get time column
153
+ df_time = df["time"]
154
+ # Split the column be parsed
155
+ df = df["TO_BE_SPLITTED"].str.split(",", expand=True, n=11)
156
+ # Assign column names
157
+ column_names = [
158
+ "station_name",
159
+ "sensor_date",
160
+ "sensor_time",
161
+ "sensor_status",
162
+ "sensor_temperature",
163
+ "number_particles",
164
+ "rainfall_rate_32bit",
165
+ "reflectivity_16bit",
166
+ "mor_visibility",
167
+ "weather_code_synop_4680",
168
+ "weather_code_synop_4677",
169
+ "raw_drop_number",
170
+ ]
171
+ df.columns = column_names
172
+ # Add time column
173
+ df["time"] = df_time
174
+ # Drop columns not needed
175
+ df = df.drop(columns=["sensor_time", "sensor_date"])
176
+ # Drop columns not agreeing with DISDRODB L0 standards
177
+ df = df.drop(columns=["station_name"])
178
+ # Return the dataframe adhering to DISDRODB L0 standards
179
+ return df
180
+
181
+ # ---------------------------------------------------------
182
+ #### Undefined number of delimiters
183
+ # - Likely a corrupted file
184
+ raise ValueError(f"Unexpected number of comma delimiters: {n_delimiters} !")
@@ -0,0 +1,113 @@
1
+ #!/usr/bin/env python3
2
+ # -----------------------------------------------------------------------------.
3
+ # Copyright (c) 2021-2023 DISDRODB developers
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify
6
+ # it under the terms of the GNU General Public License as published by
7
+ # the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+ # -----------------------------------------------------------------------------.
18
+ import pandas as pd
19
+
20
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
21
+ from disdrodb.l0.l0a_processing import read_raw_text_file
22
+
23
+
24
+ @is_documented_by(reader_generic_docstring)
25
+ def reader(
26
+ filepath,
27
+ logger=None,
28
+ ):
29
+ """Reader."""
30
+ ##------------------------------------------------------------------------.
31
+ #### Define column names
32
+ column_names = ["TO_PARSE"]
33
+
34
+ ##------------------------------------------------------------------------.
35
+ #### Define reader options
36
+ reader_kwargs = {}
37
+ # - Define delimiter
38
+ reader_kwargs["delimiter"] = "\\n"
39
+
40
+ # Skip first row as columns names
41
+ reader_kwargs["header"] = None
42
+
43
+ # - Avoid first column to become df index !!!
44
+ reader_kwargs["index_col"] = False
45
+
46
+ # - Define behaviour when encountering bad lines
47
+ reader_kwargs["on_bad_lines"] = "skip"
48
+
49
+ # - Define encoding
50
+ reader_kwargs["encoding"] = "ISO-8859-1"
51
+
52
+ # - Define reader engine
53
+ # - C engine is faster
54
+ # - Python engine is more feature-complete
55
+ reader_kwargs["engine"] = "python"
56
+
57
+ # - Define on-the-fly decompression of on-disk data
58
+ # - Available: gzip, bz2, zip
59
+ reader_kwargs["compression"] = "infer"
60
+
61
+ # - Strings to recognize as NA/NaN and replace with standard NA flags
62
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
63
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
64
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
65
+ reader_kwargs["na_values"] = ["na", "", "error"]
66
+
67
+ ##------------------------------------------------------------------------.
68
+ #### Read the data
69
+ df = read_raw_text_file(
70
+ filepath=filepath,
71
+ column_names=column_names,
72
+ reader_kwargs=reader_kwargs,
73
+ logger=logger,
74
+ )
75
+
76
+ ##------------------------------------------------------------------------.
77
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
78
+ # Retrieve file start time (date hour:minute)
79
+ start_time = df["TO_PARSE"].iloc[0]
80
+ start_time = start_time[0:16]
81
+ start_time = pd.to_datetime(start_time, format="%m/%d/%Y %H:%M", errors="coerce")
82
+ df = df.iloc[1:] # remove start_time row
83
+
84
+ # Replace heterogeneous number of spaces with ;
85
+ df["TO_PARSE"] = df["TO_PARSE"].str.replace(r" +", ";", regex=True)
86
+
87
+ # Split into columns and assign name
88
+ df = df["TO_PARSE"].str.split(";", expand=True)
89
+ columns = [
90
+ "MMSSmmm",
91
+ "rainfall_rate_32bit",
92
+ "rainfall_accumulated_32bit",
93
+ "reflectivity_32bit",
94
+ "number_particles",
95
+ "sensor_status",
96
+ "error_code",
97
+ "raw_drop_concentration",
98
+ "raw_drop_average_velocity",
99
+ "raw_drop_number",
100
+ ]
101
+ df.columns = columns
102
+
103
+ # Define datetime 'time' column
104
+ dt_minute = df["MMSSmmm"].str[0:2].astype(int).astype("<m8[m]")
105
+ dt_second = df["MMSSmmm"].str[2:4].astype(int).astype("<m8[s]")
106
+ df_time = start_time + dt_minute + dt_second
107
+ df["time"] = df_time
108
+
109
+ # Drop columns not agreeing with DISDRODB L0 standards
110
+ df = df.drop(columns=["MMSSmmm"])
111
+
112
+ # Return the dataframe adhering to DISDRODB L0 standards
113
+ return df