disdrodb 0.0.21__py3-none-any.whl → 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (279) hide show
  1. disdrodb/__init__.py +132 -15
  2. disdrodb/_config.py +4 -2
  3. disdrodb/_version.py +9 -4
  4. disdrodb/api/checks.py +264 -237
  5. disdrodb/api/configs.py +4 -8
  6. disdrodb/api/create_directories.py +235 -290
  7. disdrodb/api/info.py +217 -26
  8. disdrodb/api/io.py +306 -270
  9. disdrodb/api/path.py +597 -173
  10. disdrodb/api/search.py +486 -0
  11. disdrodb/{metadata/scripts → cli}/disdrodb_check_metadata_archive.py +12 -7
  12. disdrodb/{utils/pandas.py → cli/disdrodb_data_archive_directory.py} +9 -18
  13. disdrodb/cli/disdrodb_download_archive.py +86 -0
  14. disdrodb/cli/disdrodb_download_metadata_archive.py +53 -0
  15. disdrodb/cli/disdrodb_download_station.py +84 -0
  16. disdrodb/{api/scripts → cli}/disdrodb_initialize_station.py +22 -10
  17. disdrodb/cli/disdrodb_metadata_archive_directory.py +32 -0
  18. disdrodb/{data_transfer/scripts/disdrodb_download_station.py → cli/disdrodb_open_data_archive.py} +22 -22
  19. disdrodb/cli/disdrodb_open_logs_directory.py +69 -0
  20. disdrodb/{data_transfer/scripts/disdrodb_upload_station.py → cli/disdrodb_open_metadata_archive.py} +22 -24
  21. disdrodb/cli/disdrodb_open_metadata_directory.py +71 -0
  22. disdrodb/cli/disdrodb_open_product_directory.py +74 -0
  23. disdrodb/cli/disdrodb_open_readers_directory.py +32 -0
  24. disdrodb/{l0/scripts → cli}/disdrodb_run_l0.py +38 -31
  25. disdrodb/{l0/scripts → cli}/disdrodb_run_l0_station.py +32 -30
  26. disdrodb/{l0/scripts → cli}/disdrodb_run_l0a.py +30 -21
  27. disdrodb/{l0/scripts → cli}/disdrodb_run_l0a_station.py +24 -33
  28. disdrodb/{l0/scripts → cli}/disdrodb_run_l0b.py +30 -21
  29. disdrodb/{l0/scripts → cli}/disdrodb_run_l0b_station.py +25 -34
  30. disdrodb/cli/disdrodb_run_l0c.py +130 -0
  31. disdrodb/cli/disdrodb_run_l0c_station.py +129 -0
  32. disdrodb/cli/disdrodb_run_l1.py +122 -0
  33. disdrodb/cli/disdrodb_run_l1_station.py +121 -0
  34. disdrodb/cli/disdrodb_run_l2e.py +122 -0
  35. disdrodb/cli/disdrodb_run_l2e_station.py +122 -0
  36. disdrodb/cli/disdrodb_run_l2m.py +122 -0
  37. disdrodb/cli/disdrodb_run_l2m_station.py +122 -0
  38. disdrodb/cli/disdrodb_upload_archive.py +105 -0
  39. disdrodb/cli/disdrodb_upload_station.py +98 -0
  40. disdrodb/configs.py +90 -25
  41. disdrodb/data_transfer/__init__.py +22 -0
  42. disdrodb/data_transfer/download_data.py +87 -90
  43. disdrodb/data_transfer/upload_data.py +64 -37
  44. disdrodb/data_transfer/zenodo.py +15 -18
  45. disdrodb/docs.py +1 -1
  46. disdrodb/issue/__init__.py +17 -4
  47. disdrodb/issue/checks.py +10 -23
  48. disdrodb/issue/reader.py +9 -12
  49. disdrodb/issue/writer.py +14 -17
  50. disdrodb/l0/__init__.py +17 -26
  51. disdrodb/l0/check_configs.py +35 -23
  52. disdrodb/l0/check_standards.py +46 -51
  53. disdrodb/l0/configs/{Thies_LPM → LPM}/bins_diameter.yml +44 -44
  54. disdrodb/l0/configs/{Thies_LPM → LPM}/bins_velocity.yml +40 -40
  55. disdrodb/l0/configs/LPM/l0a_encodings.yml +80 -0
  56. disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_cf_attrs.yml +84 -65
  57. disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_encodings.yml +50 -9
  58. disdrodb/l0/configs/{Thies_LPM → LPM}/raw_data_format.yml +285 -245
  59. disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_diameter.yml +66 -66
  60. disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_velocity.yml +64 -64
  61. disdrodb/l0/configs/PARSIVEL/l0a_encodings.yml +32 -0
  62. disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_cf_attrs.yml +23 -21
  63. disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_encodings.yml +17 -17
  64. disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/raw_data_format.yml +77 -77
  65. disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_diameter.yml +64 -64
  66. disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_velocity.yml +64 -64
  67. disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +39 -0
  68. disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_cf_attrs.yml +28 -26
  69. disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_encodings.yml +20 -20
  70. disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/raw_data_format.yml +107 -107
  71. disdrodb/l0/configs/PWS100/bins_diameter.yml +173 -0
  72. disdrodb/l0/configs/PWS100/bins_velocity.yml +173 -0
  73. disdrodb/l0/configs/PWS100/l0a_encodings.yml +19 -0
  74. disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +76 -0
  75. disdrodb/l0/configs/PWS100/l0b_encodings.yml +176 -0
  76. disdrodb/l0/configs/PWS100/raw_data_format.yml +182 -0
  77. disdrodb/l0/configs/{RD_80 → RD80}/bins_diameter.yml +40 -40
  78. disdrodb/l0/configs/RD80/l0a_encodings.yml +16 -0
  79. disdrodb/l0/configs/{RD_80 → RD80}/l0b_cf_attrs.yml +3 -3
  80. disdrodb/l0/configs/RD80/l0b_encodings.yml +135 -0
  81. disdrodb/l0/configs/{RD_80 → RD80}/raw_data_format.yml +46 -50
  82. disdrodb/l0/l0_reader.py +216 -340
  83. disdrodb/l0/l0a_processing.py +237 -208
  84. disdrodb/l0/l0b_nc_processing.py +227 -80
  85. disdrodb/l0/l0b_processing.py +96 -174
  86. disdrodb/l0/l0c_processing.py +627 -0
  87. disdrodb/l0/readers/{ARM → LPM/ARM}/ARM_LPM.py +36 -58
  88. disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +236 -0
  89. disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +185 -0
  90. disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +185 -0
  91. disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +195 -0
  92. disdrodb/l0/readers/LPM/ITALY/GID_LPM_W.py +210 -0
  93. disdrodb/l0/readers/{BRAZIL/GOAMAZON_LPM.py → LPM/KIT/CHWALA.py} +97 -76
  94. disdrodb/l0/readers/LPM/SLOVENIA/ARSO.py +197 -0
  95. disdrodb/l0/readers/LPM/SLOVENIA/CRNI_VRH.py +197 -0
  96. disdrodb/l0/readers/{UK → LPM/UK}/DIVEN.py +14 -35
  97. disdrodb/l0/readers/PARSIVEL/AUSTRALIA/MELBOURNE_2007_PARSIVEL.py +157 -0
  98. disdrodb/l0/readers/PARSIVEL/CHINA/CHONGQING.py +113 -0
  99. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/ARCTIC_2021.py +40 -57
  100. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/COMMON_2011.py +37 -54
  101. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/DAVOS_2009_2011.py +34 -51
  102. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_2009.py +34 -51
  103. disdrodb/l0/readers/{EPFL/PARADISO_2014.py → PARSIVEL/EPFL/EPFL_ROOF_2008.py} +38 -50
  104. disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2010.py +105 -0
  105. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2011.py +34 -51
  106. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2012.py +33 -51
  107. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GENEPI_2007.py +25 -44
  108. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007.py +25 -44
  109. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007_2.py +25 -44
  110. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HPICONET_2010.py +34 -51
  111. disdrodb/l0/readers/{EPFL/EPFL_ROOF_2010.py → PARSIVEL/EPFL/HYMEX_LTE_SOP2.py} +37 -50
  112. disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP3.py +111 -0
  113. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HYMEX_LTE_SOP4.py +36 -54
  114. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2018.py +34 -52
  115. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2019.py +38 -56
  116. disdrodb/l0/readers/PARSIVEL/EPFL/PARADISO_2014.py +105 -0
  117. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PARSIVEL_2007.py +27 -45
  118. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PLATO_2019.py +24 -44
  119. disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019.py +140 -0
  120. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RACLETS_2019_WJF.py +41 -59
  121. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RIETHOLZBACH_2011.py +34 -51
  122. disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2017.py +117 -0
  123. disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2019.py +137 -0
  124. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/UNIL_2022.py +42 -55
  125. disdrodb/l0/readers/PARSIVEL/GPM/IFLOODS.py +104 -0
  126. disdrodb/l0/readers/{GPM → PARSIVEL/GPM}/LPVEX.py +29 -48
  127. disdrodb/l0/readers/PARSIVEL/GPM/MC3E.py +184 -0
  128. disdrodb/l0/readers/PARSIVEL/KIT/BURKINA_FASO.py +133 -0
  129. disdrodb/l0/readers/PARSIVEL/NCAR/CCOPE_2015.py +113 -0
  130. disdrodb/l0/readers/{NCAR/VORTEX_SE_2016_P1.py → PARSIVEL/NCAR/OWLES_MIPS.py} +46 -72
  131. disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +125 -0
  132. disdrodb/l0/readers/{NCAR/OWLES_MIPS.py → PARSIVEL/NCAR/PLOWS_MIPS.py} +45 -64
  133. disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +114 -0
  134. disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +176 -0
  135. disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +183 -0
  136. disdrodb/l0/readers/PARSIVEL/SLOVENIA/UL_FGG.py +121 -0
  137. disdrodb/l0/readers/{ARM/ARM_LD.py → PARSIVEL2/ARM/ARM_PARSIVEL2.py} +27 -50
  138. disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +163 -0
  139. disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +163 -0
  140. disdrodb/l0/readers/{DENMARK → PARSIVEL2/DENMARK}/EROSION_nc.py +14 -35
  141. disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +189 -0
  142. disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +119 -0
  143. disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +104 -0
  144. disdrodb/l0/readers/PARSIVEL2/GPM/NSSTC.py +176 -0
  145. disdrodb/l0/readers/PARSIVEL2/ITALY/GID_PARSIVEL2.py +32 -0
  146. disdrodb/l0/readers/PARSIVEL2/MEXICO/OH_IIUNAM_nc.py +56 -0
  147. disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +120 -0
  148. disdrodb/l0/readers/{NCAR → PARSIVEL2/NCAR}/PECAN_MIPS.py +45 -64
  149. disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +181 -0
  150. disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +160 -0
  151. disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +160 -0
  152. disdrodb/l0/readers/{NCAR/PLOWS_MIPS.py → PARSIVEL2/NCAR/VORTEX_SE_2016_P1.py} +49 -66
  153. disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +118 -0
  154. disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +152 -0
  155. disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT.py +166 -0
  156. disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +150 -0
  157. disdrodb/l0/readers/{NCAR/RELAMPAGO_RD80.py → RD80/BRAZIL/CHUVA_RD80.py} +36 -60
  158. disdrodb/l0/readers/{BRAZIL → RD80/BRAZIL}/GOAMAZON_RD80.py +36 -55
  159. disdrodb/l0/readers/{NCAR → RD80/NCAR}/CINDY_2011_RD80.py +35 -54
  160. disdrodb/l0/readers/{BRAZIL/CHUVA_RD80.py → RD80/NCAR/RELAMPAGO_RD80.py} +40 -54
  161. disdrodb/l0/readers/RD80/NOAA/PSL_RD80.py +274 -0
  162. disdrodb/l0/readers/template_reader_raw_netcdf_data.py +62 -0
  163. disdrodb/l0/readers/{reader_template.py → template_reader_raw_text_data.py} +20 -44
  164. disdrodb/l0/routines.py +885 -581
  165. disdrodb/l0/standards.py +77 -238
  166. disdrodb/l0/template_tools.py +105 -110
  167. disdrodb/l1/__init__.py +17 -0
  168. disdrodb/l1/beard_model.py +716 -0
  169. disdrodb/l1/encoding_attrs.py +635 -0
  170. disdrodb/l1/fall_velocity.py +260 -0
  171. disdrodb/l1/filters.py +192 -0
  172. disdrodb/l1/processing.py +202 -0
  173. disdrodb/l1/resampling.py +236 -0
  174. disdrodb/l1/routines.py +358 -0
  175. disdrodb/l1_env/__init__.py +17 -0
  176. disdrodb/l1_env/routines.py +38 -0
  177. disdrodb/l2/__init__.py +17 -0
  178. disdrodb/l2/empirical_dsd.py +1833 -0
  179. disdrodb/l2/event.py +388 -0
  180. disdrodb/l2/processing.py +528 -0
  181. disdrodb/l2/processing_options.py +213 -0
  182. disdrodb/l2/routines.py +868 -0
  183. disdrodb/metadata/__init__.py +9 -2
  184. disdrodb/metadata/checks.py +180 -124
  185. disdrodb/metadata/download.py +81 -0
  186. disdrodb/metadata/geolocation.py +146 -0
  187. disdrodb/metadata/info.py +20 -13
  188. disdrodb/metadata/manipulation.py +3 -3
  189. disdrodb/metadata/reader.py +59 -8
  190. disdrodb/metadata/search.py +77 -144
  191. disdrodb/metadata/standards.py +83 -80
  192. disdrodb/metadata/writer.py +10 -16
  193. disdrodb/psd/__init__.py +38 -0
  194. disdrodb/psd/fitting.py +2146 -0
  195. disdrodb/psd/models.py +774 -0
  196. disdrodb/routines.py +1412 -0
  197. disdrodb/scattering/__init__.py +28 -0
  198. disdrodb/scattering/axis_ratio.py +344 -0
  199. disdrodb/scattering/routines.py +456 -0
  200. disdrodb/utils/__init__.py +17 -0
  201. disdrodb/utils/attrs.py +208 -0
  202. disdrodb/utils/cli.py +269 -0
  203. disdrodb/utils/compression.py +60 -42
  204. disdrodb/utils/dask.py +62 -0
  205. disdrodb/utils/dataframe.py +342 -0
  206. disdrodb/utils/decorators.py +110 -0
  207. disdrodb/utils/directories.py +107 -46
  208. disdrodb/utils/encoding.py +127 -0
  209. disdrodb/utils/list.py +29 -0
  210. disdrodb/utils/logger.py +168 -46
  211. disdrodb/utils/time.py +657 -0
  212. disdrodb/utils/warnings.py +30 -0
  213. disdrodb/utils/writer.py +57 -0
  214. disdrodb/utils/xarray.py +138 -47
  215. disdrodb/utils/yaml.py +0 -1
  216. disdrodb/viz/__init__.py +17 -0
  217. disdrodb/viz/plots.py +17 -0
  218. disdrodb-0.1.1.dist-info/METADATA +294 -0
  219. disdrodb-0.1.1.dist-info/RECORD +232 -0
  220. {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info}/WHEEL +1 -1
  221. disdrodb-0.1.1.dist-info/entry_points.txt +30 -0
  222. disdrodb/data_transfer/scripts/disdrodb_download_archive.py +0 -53
  223. disdrodb/data_transfer/scripts/disdrodb_upload_archive.py +0 -57
  224. disdrodb/l0/configs/OTT_Parsivel/l0a_encodings.yml +0 -32
  225. disdrodb/l0/configs/OTT_Parsivel2/l0a_encodings.yml +0 -39
  226. disdrodb/l0/configs/RD_80/l0a_encodings.yml +0 -16
  227. disdrodb/l0/configs/RD_80/l0b_encodings.yml +0 -135
  228. disdrodb/l0/configs/Thies_LPM/l0a_encodings.yml +0 -80
  229. disdrodb/l0/io.py +0 -257
  230. disdrodb/l0/l0_processing.py +0 -1091
  231. disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_OTT.py +0 -178
  232. disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_THIES.py +0 -247
  233. disdrodb/l0/readers/BRAZIL/CHUVA_LPM.py +0 -204
  234. disdrodb/l0/readers/BRAZIL/CHUVA_OTT.py +0 -183
  235. disdrodb/l0/readers/BRAZIL/GOAMAZON_OTT.py +0 -183
  236. disdrodb/l0/readers/CHINA/CHONGQING.py +0 -131
  237. disdrodb/l0/readers/EPFL/EPFL_ROOF_2008.py +0 -128
  238. disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP2.py +0 -127
  239. disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP3.py +0 -129
  240. disdrodb/l0/readers/EPFL/RACLETS_2019.py +0 -158
  241. disdrodb/l0/readers/EPFL/SAMOYLOV_2017.py +0 -136
  242. disdrodb/l0/readers/EPFL/SAMOYLOV_2019.py +0 -158
  243. disdrodb/l0/readers/FRANCE/SIRTA_OTT2.py +0 -138
  244. disdrodb/l0/readers/GPM/GCPEX.py +0 -123
  245. disdrodb/l0/readers/GPM/IFLOODS.py +0 -123
  246. disdrodb/l0/readers/GPM/MC3E.py +0 -123
  247. disdrodb/l0/readers/GPM/NSSTC.py +0 -164
  248. disdrodb/l0/readers/ITALY/GID.py +0 -199
  249. disdrodb/l0/readers/MEXICO/OH_IIUNAM_nc.py +0 -92
  250. disdrodb/l0/readers/NCAR/CCOPE_2015.py +0 -133
  251. disdrodb/l0/readers/NCAR/PECAN_FP3.py +0 -137
  252. disdrodb/l0/readers/NCAR/PECAN_MOBILE.py +0 -144
  253. disdrodb/l0/readers/NCAR/RELAMPAGO_OTT.py +0 -195
  254. disdrodb/l0/readers/NCAR/SNOWIE_PJ.py +0 -172
  255. disdrodb/l0/readers/NCAR/SNOWIE_SB.py +0 -179
  256. disdrodb/l0/readers/NCAR/VORTEX2_2009.py +0 -133
  257. disdrodb/l0/readers/NCAR/VORTEX2_2010.py +0 -188
  258. disdrodb/l0/readers/NCAR/VORTEX2_2010_UF.py +0 -191
  259. disdrodb/l0/readers/NCAR/VORTEX_SE_2016_P2.py +0 -135
  260. disdrodb/l0/readers/NCAR/VORTEX_SE_2016_PIPS.py +0 -170
  261. disdrodb/l0/readers/NETHERLANDS/DELFT.py +0 -187
  262. disdrodb/l0/readers/SPAIN/SBEGUERIA.py +0 -179
  263. disdrodb/l0/scripts/disdrodb_run_l0b_concat.py +0 -93
  264. disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py +0 -85
  265. disdrodb/utils/netcdf.py +0 -452
  266. disdrodb/utils/scripts.py +0 -102
  267. disdrodb-0.0.21.dist-info/AUTHORS.md +0 -18
  268. disdrodb-0.0.21.dist-info/METADATA +0 -186
  269. disdrodb-0.0.21.dist-info/RECORD +0 -168
  270. disdrodb-0.0.21.dist-info/entry_points.txt +0 -15
  271. /disdrodb/l0/configs/{RD_80 → RD80}/bins_velocity.yml +0 -0
  272. /disdrodb/l0/manuals/{Thies_LPM.pdf → LPM.pdf} +0 -0
  273. /disdrodb/l0/manuals/{ODM_470.pdf → ODM470.pdf} +0 -0
  274. /disdrodb/l0/manuals/{OTT_Parsivel.pdf → PARSIVEL.pdf} +0 -0
  275. /disdrodb/l0/manuals/{OTT_Parsivel2.pdf → PARSIVEL2.pdf} +0 -0
  276. /disdrodb/l0/manuals/{PWS_100.pdf → PWS100.pdf} +0 -0
  277. /disdrodb/l0/manuals/{RD_80.pdf → RD80.pdf} +0 -0
  278. {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info/licenses}/LICENSE +0 -0
  279. {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info}/top_level.txt +0 -0
@@ -1,158 +0,0 @@
1
- #!/usr/bin/env python3
2
- # -----------------------------------------------------------------------------.
3
- # Copyright (c) 2021-2023 DISDRODB developers
4
- #
5
- # This program is free software: you can redistribute it and/or modify
6
- # it under the terms of the GNU General Public License as published by
7
- # the Free Software Foundation, either version 3 of the License, or
8
- # (at your option) any later version.
9
- #
10
- # This program is distributed in the hope that it will be useful,
11
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
- # GNU General Public License for more details.
14
- #
15
- # You should have received a copy of the GNU General Public License
16
- # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
- # -----------------------------------------------------------------------------.
18
- from disdrodb.l0 import run_l0a
19
- from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
20
-
21
-
22
- @is_documented_by(reader_generic_docstring)
23
- def reader(
24
- raw_dir,
25
- processed_dir,
26
- station_name,
27
- # Processing options
28
- force=False,
29
- verbose=False,
30
- parallel=False,
31
- debugging_mode=False,
32
- ):
33
- ##------------------------------------------------------------------------.
34
- #### - Define column names
35
- column_names = [
36
- "id",
37
- "latitude",
38
- "longitude",
39
- "time",
40
- "datalogger_temperature",
41
- "TO_BE_SPLITTED", # datalogger_voltage and rainfall_rate_32bit
42
- "rainfall_accumulated_32bit",
43
- "weather_code_synop_4680",
44
- "weather_code_synop_4677",
45
- "reflectivity_32bit",
46
- "mor_visibility",
47
- "laser_amplitude",
48
- "number_particles",
49
- "sensor_temperature",
50
- "sensor_heating_current",
51
- "sensor_battery_voltage",
52
- "sensor_status",
53
- "rainfall_amount_absolute_32bit",
54
- "error_code",
55
- "raw_drop_concentration",
56
- "raw_drop_average_velocity",
57
- "raw_drop_number",
58
- "datalogger_error",
59
- ]
60
-
61
- ##------------------------------------------------------------------------.
62
- #### - Define reader options
63
- reader_kwargs = {}
64
-
65
- # - Define delimiter
66
- reader_kwargs["delimiter"] = ";"
67
-
68
- # Skip first row as columns names
69
- reader_kwargs["header"] = None
70
-
71
- # - Avoid first column to become df index !!!
72
- reader_kwargs["index_col"] = False
73
-
74
- # - Define behaviour when encountering bad lines
75
- reader_kwargs["on_bad_lines"] = "skip"
76
-
77
- # - Define encoding
78
- # --> Avoid UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf0
79
- reader_kwargs["encoding"] = "latin-1"
80
-
81
- # - Define reader engine
82
- # - C engine is faster
83
- # - Python engine is more feature-complete
84
- reader_kwargs["engine"] = "python"
85
-
86
- # - Define on-the-fly decompression of on-disk data
87
- # - Available: gzip, bz2, zip
88
- reader_kwargs["compression"] = "infer"
89
-
90
- # - Strings to recognize as NA/NaN and replace with standard NA flags
91
- # - Already included: ‘#N/A’, ‘#N/A N/A’, ‘#NA’, ‘-1.#IND’, ‘-1.#QNAN’,
92
- # ‘-NaN’, ‘-nan’, ‘1.#IND’, ‘1.#QNAN’, ‘<NA>’, ‘N/A’,
93
- # ‘NA’, ‘NULL’, ‘NaN’, ‘n/a’, ‘nan’, ‘null’
94
- reader_kwargs["na_values"] = [
95
- "na",
96
- "",
97
- "error",
98
- "NA",
99
- ]
100
-
101
- ##------------------------------------------------------------------------.
102
- #### - Define facultative dataframe sanitizer function for L0 processing
103
- # - Enable to deal with bad raw data files
104
- # - Enable to standardize raw data files to L0 standards (i.e. time to datetime)
105
-
106
- def df_sanitizer_fun(df):
107
- # Import dask or pandas
108
- import pandas as pd
109
-
110
- # - Convert time column to datetime
111
- df["time"] = pd.to_datetime(df["time"], format="%d-%m-%Y %H:%M:%S", errors="coerce")
112
-
113
- # Drop rows with None value
114
- df = df[~df["TO_BE_SPLITTED"].isnull()]
115
-
116
- # - Drop rows when 'Error in data reading' in TO_BE_SPLITTED column
117
- bad_indices = df[df["TO_BE_SPLITTED"].str.contains("Error in data reading!")].index
118
- df = df.drop(bad_indices)
119
-
120
- # - Split TO_BE_SPLITTED columns
121
- df_splitted = df["TO_BE_SPLITTED"].str.split(",", expand=True, n=1)
122
- df_splitted.columns = ["datalogger_voltage", "rainfall_rate_32bit"]
123
- df["rainfall_rate_32bit"] = df_splitted["rainfall_rate_32bit"]
124
-
125
- # - Drop columns not agreeing with DISDRODB L0 standards
126
- columns_to_drop = [
127
- "id",
128
- "TO_BE_SPLITTED",
129
- "datalogger_temperature",
130
- "datalogger_error",
131
- "latitude",
132
- "longitude",
133
- ]
134
- df = df.drop(columns=columns_to_drop)
135
-
136
- return df
137
-
138
- ##------------------------------------------------------------------------.
139
- #### - Define glob pattern to search data files in raw_dir/data/<station_name>
140
- glob_patterns = "*.log*"
141
-
142
- ####----------------------------------------------------------------------.
143
- #### - Create L0A products
144
- run_l0a(
145
- raw_dir=raw_dir,
146
- processed_dir=processed_dir,
147
- station_name=station_name,
148
- # Custom arguments of the reader for L0A processing
149
- glob_patterns=glob_patterns,
150
- column_names=column_names,
151
- reader_kwargs=reader_kwargs,
152
- df_sanitizer_fun=df_sanitizer_fun,
153
- # Processing options
154
- force=force,
155
- verbose=verbose,
156
- parallel=parallel,
157
- debugging_mode=debugging_mode,
158
- )
@@ -1,138 +0,0 @@
1
- #!/usr/bin/env python3
2
-
3
- # -----------------------------------------------------------------------------.
4
- # Copyright (c) 2021-2023 DISDRODB developers
5
- #
6
- # This program is free software: you can redistribute it and/or modify
7
- # it under the terms of the GNU General Public License as published by
8
- # the Free Software Foundation, either version 3 of the License, or
9
- # (at your option) any later version.
10
- #
11
- # This program is distributed in the hope that it will be useful,
12
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
13
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
- # GNU General Public License for more details.
15
- #
16
- # You should have received a copy of the GNU General Public License
17
- # along with this program. If not, see <http://www.gnu.org/licenses/>.
18
- # -----------------------------------------------------------------------------.
19
- from disdrodb.l0 import run_l0a
20
- from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
21
-
22
-
23
- @is_documented_by(reader_generic_docstring)
24
- def reader(
25
- raw_dir,
26
- processed_dir,
27
- station_name,
28
- # Processing options
29
- force=False,
30
- verbose=False,
31
- parallel=False,
32
- debugging_mode=False,
33
- ):
34
- ##------------------------------------------------------------------------.
35
- #### - Define column names
36
- column_names = ["TO_SPLIT"]
37
-
38
- ##------------------------------------------------------------------------.
39
- #### - Define reader options
40
- reader_kwargs = {}
41
- # - Define delimiter
42
- reader_kwargs["delimiter"] = "\\n"
43
- # - Avoid first column to become df index !!!
44
- reader_kwargs["index_col"] = False
45
- # Skip first row as columns names
46
- reader_kwargs["header"] = None
47
- # Skip the first row (header)
48
- reader_kwargs["skiprows"] = 1
49
- # - Define behaviour when encountering bad lines
50
- reader_kwargs["on_bad_lines"] = "skip"
51
- # Define encoding
52
- reader_kwargs["encoding"] = "latin1"
53
- # - Define reader engine
54
- # - C engine is faster
55
- # - Python engine is more feature-complete
56
- reader_kwargs["engine"] = "python"
57
- # - Define on-the-fly decompression of on-disk data
58
- # - Available: gzip, bz2, zip
59
- reader_kwargs["compression"] = "infer"
60
- # - Strings to recognize as NA/NaN and replace with standard NA flags
61
- # - Already included: ‘#N/A’, ‘#N/A N/A’, ‘#NA’, ‘-1.#IND’, ‘-1.#QNAN’,
62
- # ‘-NaN’, ‘-nan’, ‘1.#IND’, ‘1.#QNAN’, ‘<NA>’, ‘N/A’,
63
- # ‘NA’, ‘NULL’, ‘NaN’, ‘n/a’, ‘nan’, ‘null’
64
- reader_kwargs["na_values"] = ["na", "", "error"]
65
-
66
- ##------------------------------------------------------------------------.
67
- #### - Define dataframe sanitizer function for L0 processing
68
- def df_sanitizer_fun(df):
69
- # Import pandas
70
- import pandas as pd
71
-
72
- # The delimiter ; is used for separating both the variables and the
73
- # values of the raw spectrum. So we need to retrieve the columns
74
- # inside the sanitizer assuming a fixed number of columns.
75
- df = df["TO_SPLIT"].str.split(";", expand=True, n=16)
76
-
77
- # Define the column names
78
- column_names = [
79
- "date",
80
- "time",
81
- "rainfall_rate_32bit",
82
- "rainfall_accumulated_32bit",
83
- "reflectivity_32bit",
84
- "mor_visibility",
85
- "laser_amplitude",
86
- "number_particles",
87
- "sensor_temperature",
88
- "sensor_heating_current",
89
- "sensor_battery_voltage",
90
- "rain_kinetic_energy",
91
- "snowfall_rate",
92
- "weather_code_synop_4680",
93
- "weather_code_metar_4678",
94
- "weather_code_nws",
95
- "raw_drop_number",
96
- ]
97
- df.columns = column_names
98
-
99
- # Define the time column
100
- df["time"] = df["date"] + "-" + df["time"]
101
- df["time"] = pd.to_datetime(df["time"], format="%Y/%m/%d-%H:%M:%S", errors="coerce")
102
- df = df.drop(columns=["date"])
103
-
104
- # Preprocess the raw spectrum
105
- # - The '<SPECTRUM>ZERO</SPECTRUM>' indicates no drops detected
106
- # --> "" generates an array of zeros in L0B processing
107
- df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>ZERO</SPECTRUM>", "")
108
- # Remove <SPECTRUM> and </SPECTRUM>" acronyms from the raw_drop_number field
109
- df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
110
- df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
111
-
112
- # Add 0 before every ; if ; not preceded by a digit
113
- # Example: ';;1;;' --> '0;0;1;0;'
114
- df["raw_drop_number"] = df["raw_drop_number"].str.replace(r"(?<!\d);", "0;", regex=True)
115
-
116
- return df
117
-
118
- ##------------------------------------------------------------------------.
119
- #### - Define glob pattern to search data files in <raw_dir>/data/<station_name>
120
- glob_patterns = "*.txt" # There is only one file without extension
121
-
122
- ####----------------------------------------------------------------------.
123
- #### - Create L0A products
124
- run_l0a(
125
- raw_dir=raw_dir,
126
- processed_dir=processed_dir,
127
- station_name=station_name,
128
- # Custom arguments of the reader for L0A processing
129
- glob_patterns=glob_patterns,
130
- column_names=column_names,
131
- reader_kwargs=reader_kwargs,
132
- df_sanitizer_fun=df_sanitizer_fun,
133
- # Processing options
134
- force=force,
135
- verbose=verbose,
136
- parallel=parallel,
137
- debugging_mode=debugging_mode,
138
- )
@@ -1,123 +0,0 @@
1
- #!/usr/bin/env python3
2
- # -----------------------------------------------------------------------------.
3
- # Copyright (c) 2021-2023 DISDRODB developers
4
- #
5
- # This program is free software: you can redistribute it and/or modify
6
- # it under the terms of the GNU General Public License as published by
7
- # the Free Software Foundation, either version 3 of the License, or
8
- # (at your option) any later version.
9
- #
10
- # This program is distributed in the hope that it will be useful,
11
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
- # GNU General Public License for more details.
14
- #
15
- # You should have received a copy of the GNU General Public License
16
- # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
- # -----------------------------------------------------------------------------.
18
- from disdrodb.l0 import run_l0a
19
- from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
20
-
21
-
22
- @is_documented_by(reader_generic_docstring)
23
- def reader(
24
- raw_dir,
25
- processed_dir,
26
- station_name,
27
- # Processing options
28
- force=False,
29
- verbose=False,
30
- parallel=False,
31
- debugging_mode=False,
32
- ):
33
- ##------------------------------------------------------------------------.
34
- #### - Define column names
35
- column_names = ["time", "TO_BE_SPLITTED"]
36
-
37
- ##------------------------------------------------------------------------.
38
- #### - Define reader options
39
- reader_kwargs = {}
40
- # Skip first row as columns names
41
- reader_kwargs["header"] = None
42
- # Skip file with encoding errors
43
- reader_kwargs["encoding_errors"] = "ignore"
44
- # - Define delimiter
45
- reader_kwargs["delimiter"] = ";"
46
- # - Avoid first column to become df index !!!
47
- reader_kwargs["index_col"] = False
48
- # - Define behaviour when encountering bad lines
49
- reader_kwargs["on_bad_lines"] = "skip"
50
- # - Define reader engine
51
- # - C engine is faster
52
- # - Python engine is more feature-complete
53
- reader_kwargs["engine"] = "python"
54
- # - Define on-the-fly decompression of on-disk data
55
- # - Available: gzip, bz2, zip
56
- reader_kwargs["compression"] = "infer"
57
- # - Strings to recognize as NA/NaN and replace with standard NA flags
58
- # - Already included: ‘#N/A’, ‘#N/A N/A’, ‘#NA’, ‘-1.#IND’, ‘-1.#QNAN’,
59
- # ‘-NaN’, ‘-nan’, ‘1.#IND’, ‘1.#QNAN’, ‘<NA>’, ‘N/A’,
60
- # ‘NA’, ‘NULL’, ‘NaN’, ‘n/a’, ‘nan’, ‘null’
61
- reader_kwargs["na_values"] = ["na", "", "error", "NA", "-.-"]
62
-
63
- ##------------------------------------------------------------------------.
64
- #### - Define dataframe sanitizer function for L0 processing
65
- def df_sanitizer_fun(df):
66
- # - Import pandas
67
- import pandas as pd
68
-
69
- # - Define 'time' datetime
70
- df_time = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
71
-
72
- # - Split the 'TO_BE_SPLITTED' column
73
- df = df["TO_BE_SPLITTED"].str.split(",", n=9, expand=True)
74
-
75
- # - Assign column names
76
- column_names = [
77
- "station_name",
78
- "sensor_status",
79
- "sensor_temperature",
80
- "number_particles",
81
- "rainfall_rate_32bit",
82
- "reflectivity_16bit",
83
- "mor_visibility",
84
- "weather_code_synop_4680",
85
- "weather_code_synop_4677",
86
- "raw_drop_number",
87
- ]
88
- df.columns = column_names
89
-
90
- # - Add the time column
91
- df["time"] = df_time
92
-
93
- # - Drop columns not agreeing with DISDRODB L0 standards
94
- df = df.drop(columns=["station_name"])
95
-
96
- # - Drop rows with invalid values
97
- # --> Ensure that weather_code_synop_4677 has length 2
98
- # --> If a previous column is missing it will have 000
99
- df = df[df["weather_code_synop_4677"].str.len() == 2]
100
-
101
- return df
102
-
103
- ##------------------------------------------------------------------------.
104
- #### - Define glob pattern to search data files in <raw_dir>/data/<station_name>
105
- glob_patterns = "*_raw.txt"
106
-
107
- ####----------------------------------------------------------------------.
108
- #### - Create L0A products
109
- run_l0a(
110
- raw_dir=raw_dir,
111
- processed_dir=processed_dir,
112
- station_name=station_name,
113
- # Custom arguments of the reader for L0A processing
114
- glob_patterns=glob_patterns,
115
- column_names=column_names,
116
- reader_kwargs=reader_kwargs,
117
- df_sanitizer_fun=df_sanitizer_fun,
118
- # Processing options
119
- force=force,
120
- verbose=verbose,
121
- parallel=parallel,
122
- debugging_mode=debugging_mode,
123
- )
@@ -1,123 +0,0 @@
1
- #!/usr/bin/env python3
2
- # -----------------------------------------------------------------------------.
3
- # Copyright (c) 2021-2023 DISDRODB developers
4
- #
5
- # This program is free software: you can redistribute it and/or modify
6
- # it under the terms of the GNU General Public License as published by
7
- # the Free Software Foundation, either version 3 of the License, or
8
- # (at your option) any later version.
9
- #
10
- # This program is distributed in the hope that it will be useful,
11
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
- # GNU General Public License for more details.
14
- #
15
- # You should have received a copy of the GNU General Public License
16
- # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
- # -----------------------------------------------------------------------------.
18
- from disdrodb.l0 import run_l0a
19
- from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
20
-
21
-
22
- @is_documented_by(reader_generic_docstring)
23
- def reader(
24
- raw_dir,
25
- processed_dir,
26
- station_name,
27
- # Processing options
28
- force=False,
29
- verbose=False,
30
- parallel=False,
31
- debugging_mode=False,
32
- ):
33
- ##------------------------------------------------------------------------.
34
- #### - Define column names
35
- column_names = ["time", "TO_BE_SPLITTED"]
36
-
37
- ##------------------------------------------------------------------------.
38
- #### - Define reader options
39
- reader_kwargs = {}
40
- # - Define delimiter
41
- reader_kwargs["delimiter"] = ";"
42
- # - Skip first row as columns names
43
- reader_kwargs["header"] = None
44
- # - Skip file with encoding errors
45
- reader_kwargs["encoding_errors"] = "ignore"
46
- # - Avoid first column to become df index !!!
47
- reader_kwargs["index_col"] = False
48
- # - Define behaviour when encountering bad lines
49
- reader_kwargs["on_bad_lines"] = "skip"
50
- # - Define reader engine
51
- # - C engine is faster
52
- # - Python engine is more feature-complete
53
- reader_kwargs["engine"] = "python"
54
- # - Define on-the-fly decompression of on-disk data
55
- # - Available: gzip, bz2, zip
56
- reader_kwargs["compression"] = "infer"
57
- # - Strings to recognize as NA/NaN and replace with standard NA flags
58
- # - Already included: ‘#N/A’, ‘#N/A N/A’, ‘#NA’, ‘-1.#IND’, ‘-1.#QNAN’,
59
- # ‘-NaN’, ‘-nan’, ‘1.#IND’, ‘1.#QNAN’, ‘<NA>’, ‘N/A’,
60
- # ‘NA’, ‘NULL’, ‘NaN’, ‘n/a’, ‘nan’, ‘null’
61
- reader_kwargs["na_values"] = ["na", "", "error", "NA", "-.-"]
62
-
63
- ##------------------------------------------------------------------------.
64
- #### - Define dataframe sanitizer function for L0 processing
65
- def df_sanitizer_fun(df):
66
- # - Import pandas
67
- import pandas as pd
68
-
69
- # - Convert time column to datetime
70
- df_time = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
71
-
72
- # - Split the 'TO_BE_SPLITTED' column
73
- df = df["TO_BE_SPLITTED"].str.split(",", expand=True, n=9)
74
-
75
- # - Assign column names
76
- columns_names = [
77
- "station_name",
78
- "sensor_status",
79
- "sensor_temperature",
80
- "number_particles",
81
- "rainfall_rate_32bit",
82
- "reflectivity_16bit",
83
- "mor_visibility",
84
- "weather_code_synop_4680",
85
- "weather_code_synop_4677",
86
- "raw_drop_number",
87
- ]
88
- df.columns = columns_names
89
-
90
- # - Add the time column
91
- df["time"] = df_time
92
-
93
- # - Drop columns not agreeing with DISDRODB L0 standards
94
- df = df.drop(columns=["station_name"])
95
-
96
- # - Drop rows with invalid values
97
- # --> Ensure that weather_code_synop_4677 has length 2
98
- # --> If a previous column is missing it will have 000
99
- df = df[df["weather_code_synop_4677"].str.len() == 2]
100
-
101
- return df
102
-
103
- ##------------------------------------------------------------------------.
104
- #### - Define glob pattern to search data files in <raw_dir>/data/<station_name>
105
- glob_patterns = "*_raw.txt"
106
-
107
- ####----------------------------------------------------------------------.
108
- #### - Create L0A products
109
- run_l0a(
110
- raw_dir=raw_dir,
111
- processed_dir=processed_dir,
112
- station_name=station_name,
113
- # Custom arguments of the reader for L0A processing
114
- glob_patterns=glob_patterns,
115
- column_names=column_names,
116
- reader_kwargs=reader_kwargs,
117
- df_sanitizer_fun=df_sanitizer_fun,
118
- # Processing options
119
- force=force,
120
- verbose=verbose,
121
- parallel=parallel,
122
- debugging_mode=debugging_mode,
123
- )
@@ -1,123 +0,0 @@
1
- #!/usr/bin/env python3
2
- # -----------------------------------------------------------------------------.
3
- # Copyright (c) 2021-2023 DISDRODB developers
4
- #
5
- # This program is free software: you can redistribute it and/or modify
6
- # it under the terms of the GNU General Public License as published by
7
- # the Free Software Foundation, either version 3 of the License, or
8
- # (at your option) any later version.
9
- #
10
- # This program is distributed in the hope that it will be useful,
11
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
- # GNU General Public License for more details.
14
- #
15
- # You should have received a copy of the GNU General Public License
16
- # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
- # -----------------------------------------------------------------------------.
18
- from disdrodb.l0 import run_l0a
19
- from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
20
-
21
-
22
- @is_documented_by(reader_generic_docstring)
23
- def reader(
24
- raw_dir,
25
- processed_dir,
26
- station_name,
27
- # Processing options
28
- force=False,
29
- verbose=False,
30
- parallel=False,
31
- debugging_mode=False,
32
- ):
33
- ##------------------------------------------------------------------------.
34
- #### - Define column name
35
- column_names = ["time", "TO_BE_SPLITTED"]
36
-
37
- ##------------------------------------------------------------------------.
38
- #### - Define reader options
39
- reader_kwargs = {}
40
- # - Define delimiter
41
- reader_kwargs["delimiter"] = ";"
42
- # - Skip first row as columns names
43
- reader_kwargs["header"] = None
44
- # - Skip file with encoding errors
45
- reader_kwargs["encoding_errors"] = "ignore"
46
- # - Avoid first column to become df index
47
- reader_kwargs["index_col"] = False
48
- # - Define behaviour when encountering bad lines
49
- reader_kwargs["on_bad_lines"] = "skip"
50
- # - Define reader engine
51
- # - C engine is faster
52
- # - Python engine is more feature-complete
53
- reader_kwargs["engine"] = "python"
54
- # - Define on-the-fly decompression of on-disk data
55
- # - Available: gzip, bz2, zip
56
- reader_kwargs["compression"] = "infer"
57
- # - Strings to recognize as NA/NaN and replace with standard NA flags
58
- # - Already included: ‘#N/A’, ‘#N/A N/A’, ‘#NA’, ‘-1.#IND’, ‘-1.#QNAN’,
59
- # ‘-NaN’, ‘-nan’, ‘1.#IND’, ‘1.#QNAN’, ‘<NA>’, ‘N/A’,
60
- # ‘NA’, ‘NULL’, ‘NaN’, ‘n/a’, ‘nan’, ‘null’
61
- reader_kwargs["na_values"] = ["na", "", "error", "-.-"]
62
-
63
- ##------------------------------------------------------------------------.
64
- #### - Define dataframe sanitizer function for L0 processing
65
- def df_sanitizer_fun(df):
66
- # - Import pandas
67
- import pandas as pd
68
-
69
- # - Define 'time' datetime
70
- df_time = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
71
-
72
- # - Split the 'TO_BE_SPLITTED' column
73
- df = df["TO_BE_SPLITTED"].str.split(",", n=9, expand=True)
74
-
75
- # - Assign column names
76
- column_names = [
77
- "station_name",
78
- "sensor_status",
79
- "sensor_temperature",
80
- "number_particles",
81
- "rainfall_rate_32bit",
82
- "reflectivity_16bit",
83
- "mor_visibility",
84
- "weather_code_synop_4680",
85
- "weather_code_synop_4677",
86
- "raw_drop_number",
87
- ]
88
- df.columns = column_names
89
-
90
- # - Add the time column
91
- df["time"] = df_time
92
-
93
- # - Drop columns not agreeing with DISDRODB L0 standards
94
- df = df.drop(columns=["station_name"])
95
-
96
- # - Drop rows with invalid values
97
- # --> Ensure that weather_code_synop_4677 has length 2
98
- # --> If a previous column is missing it will have 000
99
- df = df[df["weather_code_synop_4677"].str.len() == 2]
100
-
101
- return df
102
-
103
- ##------------------------------------------------------------------------.
104
- #### - Define glob pattern to search data files in <raw_dir>/data/<station_name>
105
- glob_patterns = "*_raw.txt"
106
-
107
- ####----------------------------------------------------------------------.
108
- #### - Create L0A products
109
- run_l0a(
110
- raw_dir=raw_dir,
111
- processed_dir=processed_dir,
112
- station_name=station_name,
113
- # Custom arguments of the reader for L0A processing
114
- glob_patterns=glob_patterns,
115
- column_names=column_names,
116
- reader_kwargs=reader_kwargs,
117
- df_sanitizer_fun=df_sanitizer_fun,
118
- # Processing options
119
- force=force,
120
- verbose=verbose,
121
- parallel=parallel,
122
- debugging_mode=debugging_mode,
123
- )