disdrodb 0.0.21__py3-none-any.whl → 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (264) hide show
  1. disdrodb/__init__.py +132 -15
  2. disdrodb/_config.py +4 -2
  3. disdrodb/_version.py +9 -4
  4. disdrodb/api/checks.py +264 -237
  5. disdrodb/api/configs.py +4 -8
  6. disdrodb/api/create_directories.py +235 -290
  7. disdrodb/api/info.py +217 -26
  8. disdrodb/api/io.py +295 -269
  9. disdrodb/api/path.py +597 -173
  10. disdrodb/api/search.py +486 -0
  11. disdrodb/{metadata/scripts → cli}/disdrodb_check_metadata_archive.py +12 -7
  12. disdrodb/{utils/pandas.py → cli/disdrodb_data_archive_directory.py} +9 -18
  13. disdrodb/cli/disdrodb_download_archive.py +86 -0
  14. disdrodb/cli/disdrodb_download_metadata_archive.py +53 -0
  15. disdrodb/cli/disdrodb_download_station.py +84 -0
  16. disdrodb/{api/scripts → cli}/disdrodb_initialize_station.py +22 -10
  17. disdrodb/cli/disdrodb_metadata_archive_directory.py +32 -0
  18. disdrodb/{data_transfer/scripts/disdrodb_download_station.py → cli/disdrodb_open_data_archive.py} +22 -22
  19. disdrodb/cli/disdrodb_open_logs_directory.py +69 -0
  20. disdrodb/{data_transfer/scripts/disdrodb_upload_station.py → cli/disdrodb_open_metadata_archive.py} +22 -24
  21. disdrodb/cli/disdrodb_open_metadata_directory.py +71 -0
  22. disdrodb/cli/disdrodb_open_product_directory.py +74 -0
  23. disdrodb/cli/disdrodb_open_readers_directory.py +32 -0
  24. disdrodb/{l0/scripts → cli}/disdrodb_run_l0.py +38 -31
  25. disdrodb/{l0/scripts → cli}/disdrodb_run_l0_station.py +32 -30
  26. disdrodb/{l0/scripts → cli}/disdrodb_run_l0a.py +30 -21
  27. disdrodb/{l0/scripts → cli}/disdrodb_run_l0a_station.py +24 -33
  28. disdrodb/{l0/scripts → cli}/disdrodb_run_l0b.py +30 -21
  29. disdrodb/{l0/scripts → cli}/disdrodb_run_l0b_station.py +25 -34
  30. disdrodb/cli/disdrodb_run_l0c.py +130 -0
  31. disdrodb/cli/disdrodb_run_l0c_station.py +129 -0
  32. disdrodb/cli/disdrodb_run_l1.py +122 -0
  33. disdrodb/cli/disdrodb_run_l1_station.py +121 -0
  34. disdrodb/cli/disdrodb_run_l2e.py +122 -0
  35. disdrodb/cli/disdrodb_run_l2e_station.py +122 -0
  36. disdrodb/cli/disdrodb_run_l2m.py +122 -0
  37. disdrodb/cli/disdrodb_run_l2m_station.py +122 -0
  38. disdrodb/cli/disdrodb_upload_archive.py +105 -0
  39. disdrodb/cli/disdrodb_upload_station.py +98 -0
  40. disdrodb/configs.py +90 -25
  41. disdrodb/data_transfer/__init__.py +22 -0
  42. disdrodb/data_transfer/download_data.py +87 -90
  43. disdrodb/data_transfer/upload_data.py +64 -37
  44. disdrodb/data_transfer/zenodo.py +15 -18
  45. disdrodb/docs.py +1 -1
  46. disdrodb/issue/__init__.py +17 -4
  47. disdrodb/issue/checks.py +10 -23
  48. disdrodb/issue/reader.py +9 -12
  49. disdrodb/issue/writer.py +14 -17
  50. disdrodb/l0/__init__.py +17 -26
  51. disdrodb/l0/check_configs.py +35 -23
  52. disdrodb/l0/check_standards.py +32 -42
  53. disdrodb/l0/configs/{Thies_LPM → LPM}/bins_diameter.yml +44 -44
  54. disdrodb/l0/configs/{Thies_LPM → LPM}/bins_velocity.yml +40 -40
  55. disdrodb/l0/configs/LPM/l0a_encodings.yml +80 -0
  56. disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_cf_attrs.yml +62 -59
  57. disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_encodings.yml +9 -9
  58. disdrodb/l0/configs/{Thies_LPM → LPM}/raw_data_format.yml +245 -245
  59. disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_diameter.yml +66 -66
  60. disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_velocity.yml +64 -64
  61. disdrodb/l0/configs/PARSIVEL/l0a_encodings.yml +32 -0
  62. disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_cf_attrs.yml +22 -20
  63. disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_encodings.yml +17 -17
  64. disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/raw_data_format.yml +77 -77
  65. disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_diameter.yml +64 -64
  66. disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_velocity.yml +64 -64
  67. disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +39 -0
  68. disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_cf_attrs.yml +24 -22
  69. disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_encodings.yml +20 -20
  70. disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/raw_data_format.yml +98 -98
  71. disdrodb/l0/configs/{RD_80 → RD80}/bins_diameter.yml +40 -40
  72. disdrodb/l0/configs/RD80/l0a_encodings.yml +16 -0
  73. disdrodb/l0/configs/{RD_80 → RD80}/l0b_cf_attrs.yml +3 -3
  74. disdrodb/l0/configs/RD80/l0b_encodings.yml +135 -0
  75. disdrodb/l0/configs/{RD_80 → RD80}/raw_data_format.yml +48 -48
  76. disdrodb/l0/l0_reader.py +216 -340
  77. disdrodb/l0/l0a_processing.py +237 -208
  78. disdrodb/l0/l0b_nc_processing.py +227 -80
  79. disdrodb/l0/l0b_processing.py +93 -173
  80. disdrodb/l0/l0c_processing.py +627 -0
  81. disdrodb/l0/readers/{ARM → LPM/ARM}/ARM_LPM.py +36 -58
  82. disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +226 -0
  83. disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +185 -0
  84. disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +183 -0
  85. disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +179 -0
  86. disdrodb/l0/readers/{UK → LPM/UK}/DIVEN.py +14 -35
  87. disdrodb/l0/readers/PARSIVEL/AUSTRALIA/MELBOURNE_2007_PARSIVEL.py +157 -0
  88. disdrodb/l0/readers/PARSIVEL/CHINA/CHONGQING.py +113 -0
  89. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/ARCTIC_2021.py +40 -57
  90. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/COMMON_2011.py +37 -54
  91. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/DAVOS_2009_2011.py +34 -51
  92. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_2009.py +34 -51
  93. disdrodb/l0/readers/{EPFL/PARADISO_2014.py → PARSIVEL/EPFL/EPFL_ROOF_2008.py} +38 -50
  94. disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2010.py +105 -0
  95. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2011.py +34 -51
  96. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2012.py +33 -51
  97. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GENEPI_2007.py +25 -44
  98. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007.py +25 -44
  99. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007_2.py +25 -44
  100. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HPICONET_2010.py +34 -51
  101. disdrodb/l0/readers/{EPFL/EPFL_ROOF_2010.py → PARSIVEL/EPFL/HYMEX_LTE_SOP2.py} +37 -50
  102. disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP3.py +111 -0
  103. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HYMEX_LTE_SOP4.py +36 -54
  104. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2018.py +34 -52
  105. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2019.py +38 -56
  106. disdrodb/l0/readers/PARSIVEL/EPFL/PARADISO_2014.py +105 -0
  107. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PARSIVEL_2007.py +27 -45
  108. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PLATO_2019.py +24 -44
  109. disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019.py +140 -0
  110. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RACLETS_2019_WJF.py +41 -59
  111. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RIETHOLZBACH_2011.py +34 -51
  112. disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2017.py +117 -0
  113. disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2019.py +137 -0
  114. disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/UNIL_2022.py +42 -55
  115. disdrodb/l0/readers/PARSIVEL/GPM/IFLOODS.py +104 -0
  116. disdrodb/l0/readers/{GPM → PARSIVEL/GPM}/LPVEX.py +29 -48
  117. disdrodb/l0/readers/PARSIVEL/GPM/MC3E.py +184 -0
  118. disdrodb/l0/readers/PARSIVEL/NCAR/CCOPE_2015.py +113 -0
  119. disdrodb/l0/readers/{NCAR/VORTEX_SE_2016_P1.py → PARSIVEL/NCAR/OWLES_MIPS.py} +46 -72
  120. disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +125 -0
  121. disdrodb/l0/readers/{NCAR/OWLES_MIPS.py → PARSIVEL/NCAR/PLOWS_MIPS.py} +45 -64
  122. disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +114 -0
  123. disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +176 -0
  124. disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +183 -0
  125. disdrodb/l0/readers/{ARM/ARM_LD.py → PARSIVEL2/ARM/ARM_PARSIVEL2.py} +27 -50
  126. disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +163 -0
  127. disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +163 -0
  128. disdrodb/l0/readers/{DENMARK → PARSIVEL2/DENMARK}/EROSION_nc.py +14 -35
  129. disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +119 -0
  130. disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +104 -0
  131. disdrodb/l0/readers/PARSIVEL2/GPM/NSSTC.py +176 -0
  132. disdrodb/l0/readers/PARSIVEL2/ITALY/GID_PARSIVEL2.py +32 -0
  133. disdrodb/l0/readers/PARSIVEL2/MEXICO/OH_IIUNAM_nc.py +56 -0
  134. disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +120 -0
  135. disdrodb/l0/readers/{NCAR → PARSIVEL2/NCAR}/PECAN_MIPS.py +45 -64
  136. disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +181 -0
  137. disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +160 -0
  138. disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +160 -0
  139. disdrodb/l0/readers/{NCAR/PLOWS_MIPS.py → PARSIVEL2/NCAR/VORTEX_SE_2016_P1.py} +49 -66
  140. disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +118 -0
  141. disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +152 -0
  142. disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT.py +166 -0
  143. disdrodb/l0/readers/{NCAR/RELAMPAGO_RD80.py → RD80/BRAZIL/CHUVA_RD80.py} +36 -60
  144. disdrodb/l0/readers/{BRAZIL → RD80/BRAZIL}/GOAMAZON_RD80.py +36 -55
  145. disdrodb/l0/readers/{NCAR → RD80/NCAR}/CINDY_2011_RD80.py +35 -54
  146. disdrodb/l0/readers/{BRAZIL/CHUVA_RD80.py → RD80/NCAR/RELAMPAGO_RD80.py} +40 -54
  147. disdrodb/l0/readers/template_reader_raw_netcdf_data.py +62 -0
  148. disdrodb/l0/readers/{reader_template.py → template_reader_raw_text_data.py} +20 -44
  149. disdrodb/l0/routines.py +885 -581
  150. disdrodb/l0/standards.py +72 -236
  151. disdrodb/l0/template_tools.py +104 -109
  152. disdrodb/l1/__init__.py +17 -0
  153. disdrodb/l1/beard_model.py +716 -0
  154. disdrodb/l1/encoding_attrs.py +620 -0
  155. disdrodb/l1/fall_velocity.py +260 -0
  156. disdrodb/l1/filters.py +192 -0
  157. disdrodb/l1/processing.py +200 -0
  158. disdrodb/l1/resampling.py +236 -0
  159. disdrodb/l1/routines.py +357 -0
  160. disdrodb/l1_env/__init__.py +17 -0
  161. disdrodb/l1_env/routines.py +38 -0
  162. disdrodb/l2/__init__.py +17 -0
  163. disdrodb/l2/empirical_dsd.py +1735 -0
  164. disdrodb/l2/event.py +388 -0
  165. disdrodb/l2/processing.py +519 -0
  166. disdrodb/l2/processing_options.py +213 -0
  167. disdrodb/l2/routines.py +868 -0
  168. disdrodb/metadata/__init__.py +9 -2
  169. disdrodb/metadata/checks.py +165 -118
  170. disdrodb/metadata/download.py +81 -0
  171. disdrodb/metadata/geolocation.py +146 -0
  172. disdrodb/metadata/info.py +20 -13
  173. disdrodb/metadata/manipulation.py +1 -1
  174. disdrodb/metadata/reader.py +59 -8
  175. disdrodb/metadata/search.py +77 -144
  176. disdrodb/metadata/standards.py +7 -8
  177. disdrodb/metadata/writer.py +8 -14
  178. disdrodb/psd/__init__.py +38 -0
  179. disdrodb/psd/fitting.py +2146 -0
  180. disdrodb/psd/models.py +774 -0
  181. disdrodb/routines.py +1176 -0
  182. disdrodb/scattering/__init__.py +28 -0
  183. disdrodb/scattering/axis_ratio.py +344 -0
  184. disdrodb/scattering/routines.py +456 -0
  185. disdrodb/utils/__init__.py +17 -0
  186. disdrodb/utils/attrs.py +208 -0
  187. disdrodb/utils/cli.py +269 -0
  188. disdrodb/utils/compression.py +60 -42
  189. disdrodb/utils/dask.py +62 -0
  190. disdrodb/utils/decorators.py +110 -0
  191. disdrodb/utils/directories.py +107 -46
  192. disdrodb/utils/encoding.py +127 -0
  193. disdrodb/utils/list.py +29 -0
  194. disdrodb/utils/logger.py +168 -46
  195. disdrodb/utils/time.py +657 -0
  196. disdrodb/utils/warnings.py +30 -0
  197. disdrodb/utils/writer.py +57 -0
  198. disdrodb/utils/xarray.py +138 -47
  199. disdrodb/utils/yaml.py +0 -1
  200. disdrodb/viz/__init__.py +17 -0
  201. disdrodb/viz/plots.py +17 -0
  202. disdrodb-0.1.0.dist-info/METADATA +321 -0
  203. disdrodb-0.1.0.dist-info/RECORD +216 -0
  204. {disdrodb-0.0.21.dist-info → disdrodb-0.1.0.dist-info}/WHEEL +1 -1
  205. disdrodb-0.1.0.dist-info/entry_points.txt +30 -0
  206. disdrodb/data_transfer/scripts/disdrodb_download_archive.py +0 -53
  207. disdrodb/data_transfer/scripts/disdrodb_upload_archive.py +0 -57
  208. disdrodb/l0/configs/OTT_Parsivel/l0a_encodings.yml +0 -32
  209. disdrodb/l0/configs/OTT_Parsivel2/l0a_encodings.yml +0 -39
  210. disdrodb/l0/configs/RD_80/l0a_encodings.yml +0 -16
  211. disdrodb/l0/configs/RD_80/l0b_encodings.yml +0 -135
  212. disdrodb/l0/configs/Thies_LPM/l0a_encodings.yml +0 -80
  213. disdrodb/l0/io.py +0 -257
  214. disdrodb/l0/l0_processing.py +0 -1091
  215. disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_OTT.py +0 -178
  216. disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_THIES.py +0 -247
  217. disdrodb/l0/readers/BRAZIL/CHUVA_LPM.py +0 -204
  218. disdrodb/l0/readers/BRAZIL/CHUVA_OTT.py +0 -183
  219. disdrodb/l0/readers/BRAZIL/GOAMAZON_LPM.py +0 -204
  220. disdrodb/l0/readers/BRAZIL/GOAMAZON_OTT.py +0 -183
  221. disdrodb/l0/readers/CHINA/CHONGQING.py +0 -131
  222. disdrodb/l0/readers/EPFL/EPFL_ROOF_2008.py +0 -128
  223. disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP2.py +0 -127
  224. disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP3.py +0 -129
  225. disdrodb/l0/readers/EPFL/RACLETS_2019.py +0 -158
  226. disdrodb/l0/readers/EPFL/SAMOYLOV_2017.py +0 -136
  227. disdrodb/l0/readers/EPFL/SAMOYLOV_2019.py +0 -158
  228. disdrodb/l0/readers/FRANCE/SIRTA_OTT2.py +0 -138
  229. disdrodb/l0/readers/GPM/GCPEX.py +0 -123
  230. disdrodb/l0/readers/GPM/IFLOODS.py +0 -123
  231. disdrodb/l0/readers/GPM/MC3E.py +0 -123
  232. disdrodb/l0/readers/GPM/NSSTC.py +0 -164
  233. disdrodb/l0/readers/ITALY/GID.py +0 -199
  234. disdrodb/l0/readers/MEXICO/OH_IIUNAM_nc.py +0 -92
  235. disdrodb/l0/readers/NCAR/CCOPE_2015.py +0 -133
  236. disdrodb/l0/readers/NCAR/PECAN_FP3.py +0 -137
  237. disdrodb/l0/readers/NCAR/PECAN_MOBILE.py +0 -144
  238. disdrodb/l0/readers/NCAR/RELAMPAGO_OTT.py +0 -195
  239. disdrodb/l0/readers/NCAR/SNOWIE_PJ.py +0 -172
  240. disdrodb/l0/readers/NCAR/SNOWIE_SB.py +0 -179
  241. disdrodb/l0/readers/NCAR/VORTEX2_2009.py +0 -133
  242. disdrodb/l0/readers/NCAR/VORTEX2_2010.py +0 -188
  243. disdrodb/l0/readers/NCAR/VORTEX2_2010_UF.py +0 -191
  244. disdrodb/l0/readers/NCAR/VORTEX_SE_2016_P2.py +0 -135
  245. disdrodb/l0/readers/NCAR/VORTEX_SE_2016_PIPS.py +0 -170
  246. disdrodb/l0/readers/NETHERLANDS/DELFT.py +0 -187
  247. disdrodb/l0/readers/SPAIN/SBEGUERIA.py +0 -179
  248. disdrodb/l0/scripts/disdrodb_run_l0b_concat.py +0 -93
  249. disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py +0 -85
  250. disdrodb/utils/netcdf.py +0 -452
  251. disdrodb/utils/scripts.py +0 -102
  252. disdrodb-0.0.21.dist-info/AUTHORS.md +0 -18
  253. disdrodb-0.0.21.dist-info/METADATA +0 -186
  254. disdrodb-0.0.21.dist-info/RECORD +0 -168
  255. disdrodb-0.0.21.dist-info/entry_points.txt +0 -15
  256. /disdrodb/l0/configs/{RD_80 → RD80}/bins_velocity.yml +0 -0
  257. /disdrodb/l0/manuals/{Thies_LPM.pdf → LPM.pdf} +0 -0
  258. /disdrodb/l0/manuals/{ODM_470.pdf → ODM470.pdf} +0 -0
  259. /disdrodb/l0/manuals/{OTT_Parsivel.pdf → PARSIVEL.pdf} +0 -0
  260. /disdrodb/l0/manuals/{OTT_Parsivel2.pdf → PARSIVEL2.pdf} +0 -0
  261. /disdrodb/l0/manuals/{PWS_100.pdf → PWS100.pdf} +0 -0
  262. /disdrodb/l0/manuals/{RD_80.pdf → RD80.pdf} +0 -0
  263. {disdrodb-0.0.21.dist-info → disdrodb-0.1.0.dist-info/licenses}/LICENSE +0 -0
  264. {disdrodb-0.0.21.dist-info → disdrodb-0.1.0.dist-info}/top_level.txt +0 -0
disdrodb/l0/routines.py CHANGED
@@ -16,702 +16,1006 @@
16
16
  # You should have received a copy of the GNU General Public License
17
17
  # along with this program. If not, see <http://www.gnu.org/licenses/>.
18
18
  # -----------------------------------------------------------------------------.
19
- """Implement DISDRODB wrappers to launch L0 processing in the terminal."""
19
+ """Implement DISDRODB L0 processing."""
20
20
 
21
21
  import datetime
22
22
  import logging
23
+ import os
23
24
  import time
25
+ from typing import Optional
24
26
 
25
- import click
27
+ import dask
26
28
 
29
+ from disdrodb.api.checks import check_sensor_name
30
+
31
+ # Directory
32
+ from disdrodb.api.create_directories import (
33
+ create_l0_directory_structure,
34
+ create_logs_directory,
35
+ create_product_directory,
36
+ )
37
+ from disdrodb.api.io import find_files, remove_product
38
+ from disdrodb.api.path import (
39
+ define_file_folder_path,
40
+ define_l0a_filename,
41
+ define_l0b_filename,
42
+ define_l0c_filename,
43
+ define_metadata_filepath,
44
+ )
45
+ from disdrodb.api.search import get_required_product
46
+ from disdrodb.configs import get_data_archive_dir, get_folder_partitioning, get_metadata_archive_dir
47
+ from disdrodb.issue import read_station_issue
48
+ from disdrodb.l0.l0_reader import get_reader
49
+ from disdrodb.l0.l0a_processing import (
50
+ read_l0a_dataframe,
51
+ sanitize_df,
52
+ write_l0a,
53
+ )
54
+ from disdrodb.l0.l0b_nc_processing import sanitize_ds
55
+ from disdrodb.l0.l0b_processing import (
56
+ create_l0b_from_l0a,
57
+ set_l0b_encodings,
58
+ write_l0b,
59
+ )
60
+ from disdrodb.l0.l0c_processing import (
61
+ create_daily_file,
62
+ get_files_per_days,
63
+ retrieve_possible_measurement_intervals,
64
+ )
65
+ from disdrodb.metadata import read_station_metadata
66
+ from disdrodb.utils.decorators import delayed_if_parallel, single_threaded_if_parallel
67
+
68
+ # Logger
27
69
  from disdrodb.utils.logger import (
28
- # log_warning,
29
- # log_error,
70
+ close_logger,
71
+ create_logger_file,
72
+ create_product_logs,
73
+ log_error,
30
74
  log_info,
31
75
  )
32
- from disdrodb.utils.scripts import _execute_cmd
76
+
77
+ # log_warning,
78
+ from disdrodb.utils.writer import write_product
79
+ from disdrodb.utils.yaml import read_yaml
33
80
 
34
81
  logger = logging.getLogger(__name__)
35
82
 
36
- ####--------------------------------------------------------------------------.
37
- #### CLIck
83
+ # -----------------------------------------------------------------------------.
84
+ #### Creation of L0A and L0B Single Station File
38
85
 
39
86
 
40
- def click_l0_stations_options(function: object):
41
- """Click command line options for DISDRODB archive L0 processing.
87
+ @delayed_if_parallel
88
+ @single_threaded_if_parallel
89
+ def _generate_l0a(
90
+ filepath,
91
+ data_dir,
92
+ logs_dir,
93
+ campaign_name,
94
+ station_name,
95
+ # Processing info
96
+ reader,
97
+ metadata,
98
+ issue_dict,
99
+ # Processing options
100
+ force,
101
+ verbose,
102
+ parallel,
103
+ ):
104
+ """Generate L0A file from raw file."""
105
+ # Define product
106
+ product = "L0A"
107
+
108
+ # Define folder partitioning
109
+ folder_partitioning = get_folder_partitioning()
110
+
111
+ # Retrieve sensor name
112
+ sensor_name = metadata["sensor_name"]
113
+
114
+ ##------------------------------------------------------------------------.
115
+ # Create file logger
116
+ filename = os.path.basename(filepath)
117
+ logger, logger_filepath = create_logger_file(
118
+ logs_dir=logs_dir,
119
+ filename=filename,
120
+ parallel=parallel,
121
+ )
42
122
 
43
- Parameters
44
- ----------
45
- function : object
46
- Function.
47
- """
48
- function = click.option(
49
- "--data_sources",
50
- type=str,
51
- show_default=True,
52
- default="",
53
- help="DISDRODB data sources to process",
54
- )(function)
55
- function = click.option(
56
- "--campaign_names",
57
- type=str,
58
- show_default=True,
59
- default="",
60
- help="DISDRODB campaign names to process",
61
- )(function)
62
- function = click.option(
63
- "--station_names",
64
- type=str,
65
- show_default=True,
66
- default="",
67
- help="DISDRODB station names to process",
68
- )(function)
69
- return function
70
-
71
-
72
- def click_l0_processing_options(function: object):
73
- """Click command line default parameters for L0 processing options.
123
+ ##------------------------------------------------------------------------.
124
+ # Log start processing
125
+ msg = f"{product} processing of {filename} has started."
126
+ log_info(logger=logger, msg=msg, verbose=verbose)
74
127
 
75
- Parameters
76
- ----------
77
- function : object
78
- Function.
79
- """
80
- function = click.option(
81
- "-p",
82
- "--parallel",
83
- type=bool,
84
- show_default=True,
85
- default=False,
86
- help="Process files in parallel",
87
- )(function)
88
- function = click.option(
89
- "-d",
90
- "--debugging_mode",
91
- type=bool,
92
- show_default=True,
93
- default=False,
94
- help="Switch to debugging mode",
95
- )(function)
96
- function = click.option("-v", "--verbose", type=bool, show_default=True, default=True, help="Verbose")(function)
97
- function = click.option(
98
- "-f",
99
- "--force",
100
- type=bool,
101
- show_default=True,
102
- default=False,
103
- help="Force overwriting",
104
- )(function)
105
- return function
106
-
107
-
108
- def click_remove_l0a_option(function: object):
109
- function = click.option(
110
- "--remove_l0a",
111
- type=bool,
112
- show_default=True,
113
- default=False,
114
- help="If true, remove the L0A files once the L0B processing is terminated.",
115
- )(function)
116
- return function
117
-
118
-
119
- def click_l0_archive_options(function: object):
120
- """Click command line arguments for L0 processing archiving of a station.
128
+ ##------------------------------------------------------------------------.
129
+ ### - Read raw file into a dataframe and sanitize for L0A format
130
+ try:
131
+ df = reader(filepath, logger=logger)
132
+ df = sanitize_df(
133
+ df=df,
134
+ sensor_name=sensor_name,
135
+ verbose=verbose,
136
+ issue_dict=issue_dict,
137
+ logger=logger,
138
+ )
121
139
 
122
- Parameters
123
- ----------
124
- function : object
125
- Function.
126
- """
127
- function = click.option(
128
- "--l0b_concat",
129
- type=bool,
130
- show_default=True,
131
- default=False,
132
- help="Produce single L0B netCDF file.",
133
- )(function)
134
- function = click.option(
135
- "--remove_l0b",
136
- type=bool,
137
- show_default=True,
138
- default=False,
139
- help="If true, remove all source L0B files once L0B concatenation is terminated.",
140
- )(function)
141
- function = click.option(
142
- "--remove_l0a",
143
- type=bool,
144
- show_default=True,
145
- default=False,
146
- help="If true, remove the L0A files once the L0B processing is terminated.",
147
- )(function)
148
- function = click.option(
149
- "-l0b",
150
- "--l0b_processing",
151
- type=bool,
152
- show_default=True,
153
- default=True,
154
- help="Perform L0B processing.",
155
- )(function)
156
- function = click.option(
157
- "-l0a",
158
- "--l0a_processing",
159
- type=bool,
160
- show_default=True,
161
- default=True,
162
- help="Perform L0A processing.",
163
- )(function)
164
- return function
165
-
166
-
167
- def click_l0b_concat_options(function: object):
168
- """Click command line default parameters for L0B concatenation.
140
+ ##--------------------------------------------------------------------.
141
+ #### - Write to Parquet
142
+ filename = define_l0a_filename(df=df, campaign_name=campaign_name, station_name=station_name)
143
+ folder_path = define_file_folder_path(df, data_dir=data_dir, folder_partitioning=folder_partitioning)
144
+ filepath = os.path.join(folder_path, filename)
145
+ write_l0a(df=df, filepath=filepath, force=force, logger=logger, verbose=verbose)
169
146
 
170
- Parameters
171
- ----------
172
- function : object
173
- Function.
174
- """
175
- function = click.option(
176
- "--remove_l0b",
177
- type=bool,
178
- show_default=True,
179
- default=False,
180
- help="If true, remove all source L0B files once L0B concatenation is terminated.",
181
- )(function)
182
- function = click.option("-v", "--verbose", type=bool, show_default=True, default=False, help="Verbose")(function)
183
- return function
147
+ ##--------------------------------------------------------------------.
148
+ # Clean environment
149
+ del df
184
150
 
151
+ # Log end processing
152
+ msg = f"{product} processing of {filename} has ended."
153
+ log_info(logger=logger, msg=msg, verbose=verbose)
185
154
 
186
- ####--------------------------------------------------------------------------.
187
- #### Run L0A and L0B Station processing
155
+ # Otherwise log the error
156
+ except Exception as e:
157
+ error_type = str(type(e).__name__)
158
+ msg = f"{error_type}: {e}"
159
+ log_error(logger=logger, msg=msg, verbose=verbose)
188
160
 
161
+ # Close the file logger
162
+ close_logger(logger)
189
163
 
190
- def run_disdrodb_l0a_station(
191
- # Station arguments
192
- data_source,
164
+ # Return the logger file path
165
+ return logger_filepath
166
+
167
+
168
+ @delayed_if_parallel
169
+ @single_threaded_if_parallel
170
+ def _generate_l0b(
171
+ filepath,
172
+ data_dir,
173
+ logs_dir,
193
174
  campaign_name,
194
175
  station_name,
176
+ # Processing info
177
+ metadata,
195
178
  # Processing options
196
- force: bool = False,
197
- verbose: bool = False,
198
- debugging_mode: bool = False,
199
- parallel: bool = True,
200
- base_dir: str = None,
179
+ force,
180
+ verbose,
181
+ parallel,
182
+ debugging_mode,
201
183
  ):
202
- """Run the L0A processing of a station calling the disdrodb_l0a_station in the terminal."""
203
- # Define command
204
- cmd = " ".join([
205
- "disdrodb_run_l0a_station",
206
- # Station arguments
207
- data_source,
208
- campaign_name,
209
- station_name,
210
- # Processing options
211
- "--force",
212
- str(force),
213
- "--verbose",
214
- str(verbose),
215
- "--debugging_mode",
216
- str(debugging_mode),
217
- "--parallel",
218
- str(parallel),
219
- "--base_dir",
220
- str(base_dir),
221
- ])
222
- # Execute command
223
- _execute_cmd(cmd)
224
- return None
225
-
226
-
227
- def run_disdrodb_l0b_station(
228
- # Station arguments
229
- data_source,
184
+ # Define product
185
+ product = "L0B"
186
+
187
+ # Define folder partitioning
188
+ folder_partitioning = get_folder_partitioning()
189
+
190
+ # -----------------------------------------------------------------.
191
+ # Create file logger
192
+ filename = os.path.basename(filepath)
193
+ logger, logger_filepath = create_logger_file(
194
+ logs_dir=logs_dir,
195
+ filename=filename,
196
+ parallel=parallel,
197
+ )
198
+
199
+ ##------------------------------------------------------------------------.
200
+ # Log start processing
201
+ msg = f"{product} processing of {filename} has started."
202
+ log_info(logger=logger, msg=msg, verbose=verbose)
203
+
204
+ ##------------------------------------------------------------------------.
205
+ # Retrieve sensor name
206
+ sensor_name = metadata["sensor_name"]
207
+ check_sensor_name(sensor_name)
208
+
209
+ ##------------------------------------------------------------------------.
210
+ try:
211
+ # Read L0A Apache Parquet file
212
+ df = read_l0a_dataframe(filepath, logger=logger, verbose=verbose, debugging_mode=debugging_mode)
213
+
214
+ # -----------------------------------------------------------------.
215
+ # Create xarray Dataset
216
+ ds = create_l0b_from_l0a(df=df, metadata=metadata, logger=logger, verbose=verbose)
217
+
218
+ # -----------------------------------------------------------------.
219
+ # Write L0B netCDF4 dataset
220
+ filename = define_l0b_filename(ds=ds, campaign_name=campaign_name, station_name=station_name)
221
+ folder_path = define_file_folder_path(ds, data_dir=data_dir, folder_partitioning=folder_partitioning)
222
+ filepath = os.path.join(folder_path, filename)
223
+ write_l0b(ds, filepath=filepath, force=force)
224
+
225
+ ##--------------------------------------------------------------------.
226
+ # Clean environment
227
+ del ds, df
228
+
229
+ # Log end processing
230
+ msg = f"{product} processing of {filename} has ended."
231
+ log_info(logger=logger, msg=msg, verbose=verbose)
232
+
233
+ # Otherwise log the error
234
+ except Exception as e:
235
+ error_type = str(type(e).__name__)
236
+ msg = f"{error_type}: {e}"
237
+ log_error(logger, msg, verbose=verbose)
238
+
239
+ # Close the file logger
240
+ close_logger(logger)
241
+
242
+ # Return the logger file path
243
+ return logger_filepath
244
+
245
+
246
+ def _generate_l0b_from_nc(
247
+ filepath,
248
+ data_dir,
249
+ logs_dir,
230
250
  campaign_name,
231
251
  station_name,
252
+ # Processing info
253
+ reader,
254
+ metadata,
255
+ issue_dict,
232
256
  # Processing options
233
- force: bool = False,
234
- verbose: bool = False,
235
- debugging_mode: bool = False,
236
- parallel: bool = True,
237
- base_dir: str = None,
238
- remove_l0a: bool = False,
257
+ force,
258
+ verbose,
259
+ parallel,
239
260
  ):
240
- """Run the L0B processing of a station calling disdrodb_run_l0b_station in the terminal."""
241
- # Define command
242
- cmd = " ".join([
243
- "disdrodb_run_l0b_station",
244
- # Station arguments
245
- data_source,
246
- campaign_name,
247
- station_name,
248
- # Processing options
249
- "--force",
250
- str(force),
251
- "--verbose",
252
- str(verbose),
253
- "--debugging_mode",
254
- str(debugging_mode),
255
- "--parallel",
256
- str(parallel),
257
- "--remove_l0a",
258
- str(remove_l0a),
259
- "--base_dir",
260
- str(base_dir),
261
- ])
262
- # Execute command
263
- _execute_cmd(cmd)
264
- return None
265
-
266
-
267
- def run_disdrodb_l0b_concat_station(
268
- data_source,
261
+
262
+ # -----------------------------------------------------------------.
263
+ # Define product name
264
+ product = "L0B"
265
+
266
+ # Define folder partitioning
267
+ folder_partitioning = get_folder_partitioning()
268
+
269
+ # Retrieve sensor name
270
+ sensor_name = metadata["sensor_name"]
271
+
272
+ # -----------------------------------------------------------------.
273
+ # Create file logger
274
+ filename = os.path.basename(filepath)
275
+ logger, logger_filepath = create_logger_file(
276
+ logs_dir=logs_dir,
277
+ filename=filename,
278
+ parallel=parallel,
279
+ )
280
+
281
+ ##------------------------------------------------------------------------.
282
+ # Log start processing
283
+ msg = f"{product} processing of {filename} has started."
284
+ log_info(logger=logger, msg=msg, verbose=verbose)
285
+
286
+ ##------------------------------------------------------------------------.
287
+ ### - Read raw netCDF and sanitize for L0B format
288
+ try:
289
+ ds = reader(filepath, logger=logger)
290
+ ds = sanitize_ds(
291
+ ds=ds,
292
+ sensor_name=sensor_name,
293
+ metadata=metadata,
294
+ issue_dict=issue_dict,
295
+ verbose=verbose,
296
+ logger=logger,
297
+ )
298
+
299
+ # -----------------------------------------------------------------.
300
+ # Write L0B netCDF4 dataset
301
+ filename = define_l0b_filename(ds=ds, campaign_name=campaign_name, station_name=station_name)
302
+ folder_path = define_file_folder_path(ds, data_dir=data_dir, folder_partitioning=folder_partitioning)
303
+ filepath = os.path.join(folder_path, filename)
304
+ write_l0b(ds, filepath=filepath, force=force)
305
+
306
+ ##--------------------------------------------------------------------.
307
+ # Clean environment
308
+ del ds
309
+
310
+ # Log end processing
311
+ msg = f"L0B processing of {filename} has ended."
312
+ log_info(logger=logger, msg=msg, verbose=verbose)
313
+
314
+ # Otherwise log the error
315
+ except Exception as e:
316
+ error_type = str(type(e).__name__)
317
+ msg = f"{error_type}: {e}"
318
+ log_error(logger, msg, verbose=verbose)
319
+
320
+ # Close the file logger
321
+ close_logger(logger)
322
+
323
+ # Return the logger file path
324
+ return logger_filepath
325
+
326
+
327
+ @delayed_if_parallel
328
+ @single_threaded_if_parallel
329
+ def _generate_l0c(
330
+ day,
331
+ filepaths,
332
+ data_dir,
333
+ logs_dir,
334
+ metadata_filepath,
269
335
  campaign_name,
270
336
  station_name,
271
- remove_l0b=False,
272
- verbose=False,
273
- base_dir=None,
337
+ # Processing options
338
+ force,
339
+ verbose,
340
+ parallel, # this is used only to initialize the correct logger !
274
341
  ):
275
- """Concatenate the L0B files of a single DISDRODB station.
342
+ # -----------------------------------------------------------------.
343
+ # Define product name
344
+ product = "L0C"
345
+
346
+ # Define folder partitioning
347
+ folder_partitioning = get_folder_partitioning()
348
+
349
+ # -----------------------------------------------------------------.
350
+ # Create file logger
351
+ logger, logger_filepath = create_logger_file(
352
+ logs_dir=logs_dir,
353
+ filename=day,
354
+ parallel=parallel,
355
+ )
276
356
 
277
- This function runs the ``disdrodb_run_l0b_concat_station`` script in the terminal.
278
- """
279
- cmd = " ".join([
280
- "disdrodb_run_l0b_concat_station",
281
- data_source,
282
- campaign_name,
283
- station_name,
284
- "--remove_l0b",
285
- str(remove_l0b),
286
- "--verbose",
287
- str(verbose),
288
- "--base_dir",
289
- str(base_dir),
290
- ])
291
- _execute_cmd(cmd)
357
+ ##------------------------------------------------------------------------.
358
+ # Log start processing
359
+ msg = f"{product} processing for {day} has started."
360
+ log_info(logger=logger, msg=msg, verbose=verbose)
361
+
362
+ ##------------------------------------------------------------------------.
363
+ ### Core computation
364
+ try:
365
+ # Retrieve measurement_intervals
366
+ # - TODO: in future available from dataset
367
+ metadata = read_yaml(metadata_filepath)
368
+ measurement_intervals = retrieve_possible_measurement_intervals(metadata)
369
+
370
+ # Produce L0C datasets
371
+ dict_ds = create_daily_file(
372
+ day=day,
373
+ filepaths=filepaths,
374
+ measurement_intervals=measurement_intervals,
375
+ ensure_variables_equality=True,
376
+ logger=logger,
377
+ verbose=verbose,
378
+ )
379
+
380
+ # Write a dataset for each sample interval
381
+ for ds in dict_ds.values(): # (sample_interval, ds)
382
+ # Write L0C netCDF4 dataset
383
+ if ds["time"].size > 1:
384
+ # Get sensor name from dataset
385
+ sensor_name = ds.attrs.get("sensor_name")
386
+ campaign_name = ds.attrs.get("campaign_name")
387
+ station_name = ds.attrs.get("station_name")
388
+
389
+ # Set encodings
390
+ ds = set_l0b_encodings(ds=ds, sensor_name=sensor_name)
391
+
392
+ # Define filepath
393
+ filename = define_l0c_filename(ds, campaign_name=campaign_name, station_name=station_name)
394
+ folder_path = define_file_folder_path(ds, data_dir=data_dir, folder_partitioning=folder_partitioning)
395
+ filepath = os.path.join(folder_path, filename)
396
+
397
+ # Write to disk
398
+ write_product(ds, product=product, filepath=filepath, force=force)
399
+
400
+ # Clean environment
401
+ del ds
402
+
403
+ # Log end processing
404
+ msg = f"{product} processing for {day} has ended."
405
+ log_info(logger=logger, msg=msg, verbose=verbose)
406
+
407
+ ##--------------------------------------------------------------------.
408
+ # Otherwise log the error
409
+ except Exception as e:
410
+ error_type = str(type(e).__name__)
411
+ msg = f"{error_type}: {e}"
412
+ log_error(logger, msg, verbose=verbose)
413
+
414
+ # Close the file logger
415
+ close_logger(logger)
416
+
417
+ # Return the logger file path
418
+ return logger_filepath
292
419
 
293
420
 
294
421
  ####--------------------------------------------------------------------------.
295
- #### Run L0 Station processing (L0A + L0B)
422
+ #### DISDRODB Station Functions
296
423
 
297
424
 
298
- def run_disdrodb_l0_station(
425
+ def run_l0a_station(
426
+ # Station arguments
299
427
  data_source,
300
428
  campaign_name,
301
429
  station_name,
302
- # L0 archive options
303
- l0a_processing: bool = True,
304
- l0b_processing: bool = True,
305
- l0b_concat: bool = True,
306
- remove_l0a: bool = False,
307
- remove_l0b: bool = False,
308
430
  # Processing options
309
431
  force: bool = False,
310
432
  verbose: bool = False,
311
433
  debugging_mode: bool = False,
312
434
  parallel: bool = True,
313
- base_dir: str = None,
435
+ # DISDRODB root directories
436
+ data_archive_dir: Optional[str] = None,
437
+ metadata_archive_dir: Optional[str] = None,
314
438
  ):
315
- """Run the L0 processing of a specific DISDRODB station from the terminal.
439
+ """
440
+ Run the L0A processing of a specific DISDRODB station when invoked from the terminal.
441
+
442
+ This function is intended to be called through the ``disdrodb_run_l0a_station``
443
+ command-line interface.
316
444
 
317
445
  Parameters
318
446
  ----------
319
447
  data_source : str
320
- Institution name (when campaign data spans more than 1 country),
321
- or country (when all campaigns (or sensor networks) are inside a given country).
322
- Must be UPPER CASE.
448
+ The name of the institution (for campaigns spanning multiple countries) or
449
+ the name of the country (for campaigns or sensor networks within a single country).
450
+ Must be provided in UPPER CASE.
323
451
  campaign_name : str
324
- Campaign name. Must be UPPER CASE.
452
+ The name of the campaign. Must be provided in UPPER CASE.
325
453
  station_name : str
326
- Station name
327
- l0a_processing : bool
328
- Whether to launch processing to generate L0A Apache Parquet file(s) from raw data.
329
- The default is True.
330
- l0b_processing : bool
331
- Whether to launch processing to generate L0B netCDF4 file(s) from L0A data.
332
- The default is True.
333
- l0b_concat : bool
334
- Whether to concatenate all raw files into a single L0B netCDF file.
335
- If l0b_concat=True, all raw files will be saved into a single L0B netCDF file.
336
- If l0b_concat=False, each raw file will be converted into the corresponding L0B netCDF file.
337
- The default is False.
338
- remove_l0a : bool
339
- Whether to keep the L0A files after having generated the L0B netCDF products.
340
- The default is False.
341
- remove_l0b : bool
342
- Whether to remove the L0B files after having concatenated all L0B netCDF files.
343
- It takes places only if l0b_concat=True
344
- The default is False.
345
- force : bool
346
- If True, overwrite existing data into destination directories.
347
- If False, raise an error if there are already data into destination directories.
348
- The default is False.
349
- verbose : bool
350
- Whether to print detailed processing information into terminal.
351
- The default is True.
352
- parallel : bool
353
- If True, the files are processed simultaneously in multiple processes.
354
- Each process will use a single thread to avoid issues with the HDF/netCDF library.
355
- By default, the number of process is defined with os.cpu_count().
356
- If False, the files are processed sequentially in a single process.
357
- If False, multi-threading is automatically exploited to speed up I/0 tasks.
358
- debugging_mode : bool
359
- If True, it reduces the amount of data to process.
360
- For L0A, it processes just the first 3 raw data files for each station.
361
- For L0B, it processes just the first 100 rows of 3 L0A files for each station.
362
- The default is False.
363
- base_dir : str (optional)
364
- Base directory of DISDRODB. Format: <...>/DISDRODB
365
- If None (the default), the disdrodb config variable 'dir' is used.
454
+ The name of the station.
455
+ force : bool, optional
456
+ If ``True``, existing data in the destination directories will be overwritten.
457
+ If ``False`` (default), an error will be raised if data already exists in the destination directories.
458
+ verbose : bool, optional
459
+ If ``True`` (default), detailed processing information will be printed to the terminal.
460
+ If ``False``, less information will be displayed.
461
+ parallel : bool, optional
462
+ If ``True``, files will be processed in multiple processes simultaneously
463
+ with each process using a single thread.
464
+ If ``False`` (default), files will be processed sequentially in a single process,
465
+ and multi-threading will be automatically exploited to speed up I/O tasks.
466
+ debugging_mode : bool, optional
467
+ If ``True``, the amount of data processed will be reduced.
468
+ Only the first 3 raw data files will be processed. The default value is ``False``.
469
+ data_archive_dir : str, optional
470
+ The base directory of DISDRODB, expected in the format ``<...>/DISDRODB``.
471
+ If not specified, the path specified in the DISDRODB active configuration will be used.
366
472
  """
473
+ # Retrieve DISDRODB Metadata Archive and Data Archive root directories
474
+ data_archive_dir = get_data_archive_dir(data_archive_dir)
475
+ metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
476
+
477
+ # Read metadata
478
+ metadata = read_station_metadata(
479
+ metadata_archive_dir=metadata_archive_dir,
480
+ data_source=data_source,
481
+ campaign_name=campaign_name,
482
+ station_name=station_name,
483
+ )
367
484
 
368
- # ---------------------------------------------------------------------.
485
+ # ------------------------------------------------------------------------.
486
+ # Define raw data ingestion chain
487
+ # --> If raw data are netCDF files, this routine produces directly L0B files
488
+ # --> Otherwise, it produces L0A files.
489
+ if metadata["raw_data_format"] == "netcdf":
490
+ generate_standardized_files = _generate_l0b_from_nc
491
+ product = "L0B"
492
+ else:
493
+ generate_standardized_files = _generate_l0a
494
+ product = "L0A"
495
+
496
+ # ------------------------------------------------------------------------.
497
+ # Start product processing
369
498
  t_i = time.time()
370
- msg = f"L0 processing of station {station_name} has started."
499
+ msg = f"{product} processing of station {station_name} has started."
371
500
  log_info(logger=logger, msg=msg, verbose=verbose)
372
501
 
373
- # ------------------------------------------------------------------.
374
- # L0A processing
375
- if l0a_processing:
376
- run_disdrodb_l0a_station(
377
- # Station arguments
378
- base_dir=base_dir,
379
- data_source=data_source,
380
- campaign_name=campaign_name,
381
- station_name=station_name,
382
- # Processing options
383
- force=force,
384
- verbose=verbose,
385
- debugging_mode=debugging_mode,
386
- parallel=parallel,
387
- )
388
- # ------------------------------------------------------------------.
389
- # L0B processing
390
- if l0b_processing:
391
- run_disdrodb_l0b_station(
392
- # Station arguments
393
- base_dir=base_dir,
394
- data_source=data_source,
395
- campaign_name=campaign_name,
396
- station_name=station_name,
397
- # Processing options
398
- force=force,
399
- verbose=verbose,
400
- debugging_mode=debugging_mode,
401
- parallel=parallel,
402
- remove_l0a=remove_l0a,
403
- )
404
-
405
502
  # ------------------------------------------------------------------------.
406
- # If l0b_concat=True, concat the netCDF in a single file
407
- if l0b_concat:
408
- run_disdrodb_l0b_concat_station(
409
- base_dir=base_dir,
410
- data_source=data_source,
411
- campaign_name=campaign_name,
412
- station_name=station_name,
413
- remove_l0b=remove_l0b,
414
- verbose=verbose,
415
- )
503
+ # Create directory structure
504
+ data_dir = create_l0_directory_structure(
505
+ data_archive_dir=data_archive_dir,
506
+ data_source=data_source,
507
+ campaign_name=campaign_name,
508
+ metadata_archive_dir=metadata_archive_dir,
509
+ product=product, # L0A or L0B
510
+ station_name=station_name,
511
+ force=force,
512
+ )
416
513
 
417
514
  # -------------------------------------------------------------------------.
418
- # End of L0 processing for all stations
419
- timedelta_str = str(datetime.timedelta(seconds=time.time() - t_i))
420
- msg = f"L0 processing of stations {station_name} completed in {timedelta_str}"
421
- log_info(logger, msg, verbose)
422
- return None
515
+ # Define logs directory
516
+ logs_dir = create_logs_directory(
517
+ product=product,
518
+ data_archive_dir=data_archive_dir,
519
+ data_source=data_source,
520
+ campaign_name=campaign_name,
521
+ station_name=station_name,
522
+ )
423
523
 
524
+ # -----------------------------------------------------------------.
525
+ # Read issue YAML file
526
+ issue_dict = read_station_issue(
527
+ metadata_archive_dir=metadata_archive_dir,
528
+ data_source=data_source,
529
+ campaign_name=campaign_name,
530
+ station_name=station_name,
531
+ )
424
532
 
425
- ####---------------------------------------------------------------------------.
426
- #### Run L0 Archive processing
533
+ ##------------------------------------------------------------------------.
534
+ # Retrieve sensor name
535
+ sensor_name = metadata["sensor_name"]
536
+ check_sensor_name(sensor_name)
427
537
 
538
+ # Retrieve sensor name
539
+ reader_reference = metadata["reader"]
428
540
 
429
- def _check_available_stations(list_info):
430
- # If no stations available, raise an error
431
- if len(list_info) == 0:
432
- msg = "No stations available given the provided `data_sources` and `campaign_names` arguments !"
433
- raise ValueError(msg)
541
+ # Retrieve glob patterns
542
+ glob_pattern = metadata["raw_data_glob_pattern"]
434
543
 
544
+ ##------------------------------------------------------------------------.
545
+ # Retrieve reader
546
+ reader = get_reader(reader_reference, sensor_name=sensor_name)
435
547
 
436
- def _filter_list_info(list_info, station_names):
437
- # Filter by provided stations
438
- if station_names is not None:
439
- list_info = [info for info in list_info if info[2] in station_names]
440
- # If nothing left, raise an error
441
- if len(list_info) == 0:
442
- raise ValueError("No stations available given the provided `station_names` argument !")
443
- return list_info
548
+ # -------------------------------------------------------------------------.
549
+ # List files to process
550
+ filepaths = find_files(
551
+ data_source=data_source,
552
+ campaign_name=campaign_name,
553
+ station_name=station_name,
554
+ product="RAW",
555
+ debugging_mode=debugging_mode,
556
+ data_archive_dir=data_archive_dir,
557
+ glob_pattern=glob_pattern,
558
+ )
444
559
 
560
+ # Print the number of files to be processed
561
+ n_files = len(filepaths)
562
+ msg = f"{n_files} raw files are ready to be processed."
563
+ log_info(logger=logger, msg=msg, verbose=verbose)
445
564
 
446
- def _get_starting_product(l0a_processing, l0b_processing):
447
- if l0a_processing:
448
- product = "RAW"
449
- elif l0b_processing:
450
- product = "L0A"
451
- else:
452
- raise ValueError("At least l0a_processing or l0b_processing must be True.")
453
- return product
454
-
455
-
456
- def run_disdrodb_l0(
457
- data_sources=None,
458
- campaign_names=None,
459
- station_names=None,
460
- # L0 archive options
461
- l0a_processing: bool = True,
462
- l0b_processing: bool = True,
463
- l0b_concat: bool = False,
565
+ # -----------------------------------------------------------------.
566
+ # Generate L0A/L0B files
567
+ # - Loop over the files and save the L0A Apache Parquet files.
568
+ # - If parallel=True, it does that in parallel using dask.delayed
569
+ list_tasks = [
570
+ generate_standardized_files(
571
+ filepath=filepath,
572
+ data_dir=data_dir,
573
+ logs_dir=logs_dir,
574
+ campaign_name=campaign_name,
575
+ station_name=station_name,
576
+ # Reader argument
577
+ reader=reader,
578
+ # Processing info
579
+ metadata=metadata,
580
+ issue_dict=issue_dict,
581
+ # Processing options
582
+ force=force,
583
+ verbose=verbose,
584
+ parallel=parallel,
585
+ )
586
+ for filepath in filepaths
587
+ ]
588
+ list_logs = dask.compute(*list_tasks) if parallel else list_tasks
589
+
590
+ # -----------------------------------------------------------------.
591
+ # Define product summary logs
592
+ create_product_logs(
593
+ product=product,
594
+ data_source=data_source,
595
+ campaign_name=campaign_name,
596
+ station_name=station_name,
597
+ data_archive_dir=data_archive_dir,
598
+ # Logs list
599
+ list_logs=list_logs,
600
+ )
601
+
602
+ # ---------------------------------------------------------------------.
603
+ # End product processing
604
+ timedelta_str = str(datetime.timedelta(seconds=round(time.time() - t_i)))
605
+ msg = f"{product} processing of station {station_name} completed in {timedelta_str}"
606
+ log_info(logger=logger, msg=msg, verbose=verbose)
607
+ # ---------------------------------------------------------------------.
608
+
609
+
610
+ def run_l0b_station(
611
+ # Station arguments
612
+ data_source,
613
+ campaign_name,
614
+ station_name,
615
+ # L0B processing options
464
616
  remove_l0a: bool = False,
465
- remove_l0b: bool = False,
466
617
  # Processing options
467
618
  force: bool = False,
468
- verbose: bool = False,
469
- debugging_mode: bool = False,
619
+ verbose: bool = True,
470
620
  parallel: bool = True,
471
- base_dir: str = None,
621
+ debugging_mode: bool = False,
622
+ # DISDRODB root directories
623
+ data_archive_dir: Optional[str] = None,
624
+ metadata_archive_dir: Optional[str] = None,
472
625
  ):
473
- """Run the L0 processing of DISDRODB stations.
626
+ """
627
+ Run the L0B processing of a specific DISDRODB station when invoked from the terminal.
474
628
 
475
- This function allows to launch the processing of many DISDRODB stations with a single command.
476
- From the list of all available DISDRODB stations, it runs the processing of the
477
- stations matching the provided data_sources, campaign_names and station_names.
629
+ This function is intended to be called through the ``disdrodb_run_l0b_station``
630
+ command-line interface.
478
631
 
479
632
  Parameters
480
633
  ----------
481
- data_sources : list
482
- Name of data source(s) to process.
483
- The name(s) must be UPPER CASE.
484
- If campaign_names and station are not specified, process all stations.
485
- The default is None
486
- campaign_names : list
487
- Name of the campaign(s) to process.
488
- The name(s) must be UPPER CASE.
489
- The default is None
490
- station_names : list
491
- Station names to process.
492
- The default is None
493
- l0a_processing : bool
494
- Whether to launch processing to generate L0A Apache Parquet file(s) from raw data.
495
- The default is True.
496
- l0b_processing : bool
497
- Whether to launch processing to generate L0B netCDF4 file(s) from L0A data.
498
- The default is True.
499
- l0b_concat : bool
500
- Whether to concatenate all raw files into a single L0B netCDF file.
501
- If l0b_concat=True, all raw files will be saved into a single L0B netCDF file.
502
- If l0b_concat=False, each raw file will be converted into the corresponding L0B netCDF file.
503
- The default is False.
504
- remove_l0a : bool
505
- Whether to keep the L0A files after having generated the L0B netCDF products.
506
- The default is False.
507
- remove_l0b : bool
508
- Whether to remove the L0B files after having concatenated all L0B netCDF files.
509
- It takes places only if l0b_concat = True
510
- The default is False.
511
- force : bool
512
- If True, overwrite existing data into destination directories.
513
- If False, raise an error if there are already data into destination directories.
514
- The default is False.
515
- verbose : bool
516
- Whether to print detailed processing information into terminal.
517
- The default is True.
518
- parallel : bool
519
- If True, the files are processed simultaneously in multiple processes.
520
- Each process will use a single thread to avoid issues with the HDF/netCDF library.
521
- By default, the number of process is defined with os.cpu_count().
522
- If False, the files are processed sequentially in a single process.
523
- If False, multi-threading is automatically exploited to speed up I/0 tasks.
524
- debugging_mode : bool
525
- If True, it reduces the amount of data to process.
526
- For L0A, it processes just the first 3 raw data files.
527
- For L0B, it processes just the first 100 rows of 3 L0A files.
528
- The default is False.
529
- base_dir : str (optional)
530
- Base directory of DISDRODB. Format: <...>/DISDRODB
531
- If None (the default), the disdrodb config variable 'dir' is used.
634
+ data_source : str
635
+ The name of the institution (for campaigns spanning multiple countries) or
636
+ the name of the country (for campaigns or sensor networks within a single country).
637
+ Must be provided in UPPER CASE.
638
+ campaign_name : str
639
+ The name of the campaign. Must be provided in UPPER CASE.
640
+ station_name : str
641
+ The name of the station.
642
+ force : bool, optional
643
+ If ``True``, existing data in the destination directories will be overwritten.
644
+ If ``False`` (default), an error will be raised if data already exists in the destination directories.
645
+ verbose : bool, optional
646
+ If ``True`` (default), detailed processing information will be printed to the terminal.
647
+ If ``False``, less information will be displayed.
648
+ parallel : bool, optional
649
+ If ``True``, files will be processed in multiple processes simultaneously,
650
+ with each process using a single thread to avoid issues with the HDF/netCDF library.
651
+ If ``False`` (default), files will be processed sequentially in a single process,
652
+ and multi-threading will be automatically exploited to speed up I/O tasks.
653
+ debugging_mode : bool, optional
654
+ If ``True``, the amount of data processed will be reduced.
655
+ Only the first 100 rows of 3 L0A files will be processed. The default value is ``False``.
656
+ remove_l0a: bool, optional
657
+ Whether to remove the processed L0A files. The default value is ``False``.
658
+ data_archive_dir : str, optional
659
+ The base directory of DISDRODB, expected in the format ``<...>/DISDRODB``.
660
+ If not specified, the path specified in the DISDRODB active configuration will be used.
661
+
532
662
  """
533
- from disdrodb.api.io import available_stations
663
+ # Define product
664
+ product = "L0B"
665
+
666
+ # Retrieve DISDRODB base directory
667
+ data_archive_dir = get_data_archive_dir(data_archive_dir)
668
+
669
+ # Retrieve DISDRODB Metadata Archive directory
670
+ metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
671
+
672
+ # -----------------------------------------------------------------.
673
+ # Retrieve metadata
674
+ metadata = read_station_metadata(
675
+ metadata_archive_dir=metadata_archive_dir,
676
+ data_source=data_source,
677
+ campaign_name=campaign_name,
678
+ station_name=station_name,
679
+ )
680
+
681
+ # Skip run_l0b processing if the raw data are netCDFs
682
+ # - L0B produced when running L0A ...
683
+ if metadata["raw_data_format"] == "netcdf":
684
+ return
685
+
686
+ # -----------------------------------------------------------------.
687
+ # Start L0B processing
688
+ t_i = time.time()
689
+ msg = f"{product} processing of station_name {station_name} has started."
690
+ log_info(logger=logger, msg=msg, verbose=verbose)
534
691
 
535
- # Get list of available stations
536
- product = _get_starting_product(l0a_processing=l0a_processing, l0b_processing=l0b_processing)
537
- list_info = available_stations(
538
- base_dir=base_dir,
692
+ # -----------------------------------------------------------------.
693
+ # Define logs directory
694
+ logs_dir = create_logs_directory(
539
695
  product=product,
540
- data_sources=data_sources,
541
- campaign_names=campaign_names,
696
+ data_archive_dir=data_archive_dir,
697
+ data_source=data_source,
698
+ campaign_name=campaign_name,
699
+ station_name=station_name,
542
700
  )
543
- _check_available_stations(list_info)
544
- list_info = _filter_list_info(list_info, station_names)
545
-
546
- # Print message
547
- n_stations = len(list_info)
548
- print(f"L0 processing of {n_stations} stations started.")
549
-
550
- # Loop over stations
551
- for data_source, campaign_name, station_name in list_info:
552
- print(f"L0 processing of {data_source} {campaign_name} {station_name} station started.")
553
- # Run processing
554
- run_disdrodb_l0_station(
555
- base_dir=base_dir,
701
+
702
+ # Create product directory
703
+ data_dir = create_product_directory(
704
+ data_archive_dir=data_archive_dir,
705
+ metadata_archive_dir=metadata_archive_dir,
706
+ data_source=data_source,
707
+ campaign_name=campaign_name,
708
+ station_name=station_name,
709
+ product=product,
710
+ force=force,
711
+ )
712
+
713
+ ##----------------------------------------------------------------.
714
+ # Get L0A files for the station
715
+ required_product = get_required_product(product)
716
+ flag_not_available_data = False
717
+ try:
718
+ filepaths = find_files(
719
+ data_archive_dir=data_archive_dir,
556
720
  data_source=data_source,
557
721
  campaign_name=campaign_name,
558
722
  station_name=station_name,
559
- # L0 archive options
560
- l0a_processing=l0a_processing,
561
- l0b_processing=l0b_processing,
562
- l0b_concat=l0b_concat,
563
- remove_l0a=remove_l0a,
564
- remove_l0b=remove_l0b,
565
- # Process options
723
+ product=required_product,
724
+ debugging_mode=debugging_mode,
725
+ )
726
+ except Exception as e:
727
+ print(str(e)) # Case where no file paths available
728
+ flag_not_available_data = True
729
+
730
+ # -------------------------------------------------------------------------.
731
+ # If no data available, print error message and return None
732
+ if flag_not_available_data:
733
+ msg = (
734
+ f"{product} processing of {data_source} {campaign_name} {station_name}"
735
+ + f"has not been launched because of missing {required_product} data."
736
+ )
737
+ print(msg)
738
+ return
739
+
740
+ ##----------------------------------------------------------------.
741
+ # Generate L0B files
742
+ # Loop over the L0A files and save the L0B netCDF files.
743
+ # - If parallel=True, it does that in parallel using dask.bag
744
+ # Settings npartitions=len(filepaths) enable to wait prior task on a core
745
+ # finish before starting a new one.
746
+ # BUG: If debugging_mode=True and parallel=True a subtle bug can currently occur when
747
+ # two processes with a subsetted L0A files want to create the same L0B files !
748
+ list_tasks = [
749
+ _generate_l0b(
750
+ filepath=filepath,
751
+ data_dir=data_dir,
752
+ logs_dir=logs_dir,
753
+ metadata=metadata,
754
+ campaign_name=campaign_name,
755
+ station_name=station_name,
566
756
  force=force,
567
757
  verbose=verbose,
568
758
  debugging_mode=debugging_mode,
569
759
  parallel=parallel,
570
760
  )
571
- print(f"L0 processing of {data_source} {campaign_name} {station_name} station ended.")
761
+ for filepath in filepaths
762
+ ]
763
+ list_logs = dask.compute(*list_tasks) if parallel else list_tasks
764
+ # if not parallel:
765
+ # list_logs = [
766
+ # _generate_l0b(
767
+ # filepath=filepath,
768
+ # data_dir=data_dir,
769
+ # logs_dir=logs_dir,
770
+ # metadata=metadata,
771
+ # campaign_name=campaign_name,
772
+ # station_name=station_name,
773
+ # force=force,
774
+ # verbose=verbose,
775
+ # debugging_mode=debugging_mode,
776
+ # parallel=parallel,
777
+ # )
778
+ # for filepath in filepaths
779
+ # ]
780
+
781
+ # else:
782
+ # bag = db.from_sequence(filepaths, npartitions=len(filepaths))
783
+ # list_logs = bag.map(
784
+ # _generate_l0b,
785
+ # data_dir=data_dir,
786
+ # logs_dir=logs_dir,
787
+ # metadata=metadata,
788
+ # campaign_name=campaign_name,
789
+ # station_name=station_name,
790
+ # force=force,
791
+ # verbose=verbose,
792
+ # debugging_mode=debugging_mode,
793
+ # parallel=parallel,
794
+ # ).compute()
795
+
796
+ # -----------------------------------------------------------------.
797
+ # Define L0B summary logs
798
+ create_product_logs(
799
+ product=product,
800
+ data_source=data_source,
801
+ campaign_name=campaign_name,
802
+ station_name=station_name,
803
+ data_archive_dir=data_archive_dir,
804
+ # Logs list
805
+ list_logs=list_logs,
806
+ )
807
+
808
+ # -----------------------------------------------------------------.
809
+ # End L0B processing
810
+ timedelta_str = str(datetime.timedelta(seconds=round(time.time() - t_i)))
811
+ msg = f"{product} processing of station_name {station_name} completed in {timedelta_str}"
812
+ log_info(logger=logger, msg=msg, verbose=verbose)
813
+
814
+ # -----------------------------------------------------------------.
815
+ # Option to remove L0A
816
+ if remove_l0a:
817
+ remove_product(
818
+ data_archive_dir=data_archive_dir,
819
+ product="L0A",
820
+ data_source=data_source,
821
+ campaign_name=campaign_name,
822
+ station_name=station_name,
823
+ logger=logger,
824
+ verbose=verbose,
825
+ )
572
826
 
573
827
 
574
- def run_disdrodb_l0a(
575
- data_sources=None,
576
- campaign_names=None,
577
- station_names=None,
828
+ def run_l0c_station(
829
+ # Station arguments
830
+ data_source,
831
+ campaign_name,
832
+ station_name,
833
+ # L0C processing options
834
+ remove_l0b: bool = False,
578
835
  # Processing options
579
836
  force: bool = False,
580
- verbose: bool = False,
581
- debugging_mode: bool = False,
837
+ verbose: bool = True,
582
838
  parallel: bool = True,
583
- base_dir: str = None,
839
+ debugging_mode: bool = False,
840
+ # DISDRODB root directories
841
+ data_archive_dir: Optional[str] = None,
842
+ metadata_archive_dir: Optional[str] = None,
584
843
  ):
585
- """Run the L0A processing of DISDRODB stations.
844
+ """
845
+ Run the L0C processing of a specific DISDRODB station when invoked from the terminal.
846
+
847
+ The DISDRODB L0A and L0B routines just convert source raw data into netCDF format.
848
+ The DISDRODB L0C routine ingests L0B files and performs data homogenization.
849
+ The DISDRODB L0C routine takes care of:
850
+
851
+ - removing duplicated timesteps across files,
852
+ - merging/splitting files into daily files,
853
+ - regularizing timesteps for potentially trailing seconds,
854
+ - ensuring L0C files with unique sample intervals.
586
855
 
587
- This function allows to launch the processing of many DISDRODB stations with a single command.
588
- From the list of all available DISDRODB stations, it runs the processing of the
589
- stations matching the provided data_sources, campaign_names and station_names.
856
+ Duplicated timesteps are automatically dropped if their variable values coincides,
857
+ otherwise an error is raised.
858
+
859
+ This function is intended to be called through the ``disdrodb_run_l0c_station``
860
+ command-line interface.
590
861
 
591
862
  Parameters
592
863
  ----------
593
- data_sources : list
594
- Name of data source(s) to process.
595
- The name(s) must be UPPER CASE.
596
- If campaign_names and station are not specified, process all stations.
597
- The default is None
598
- campaign_names : list
599
- Name of the campaign(s) to process.
600
- The name(s) must be UPPER CASE.
601
- The default is None
602
- station_names : list
603
- Station names to process.
604
- The default is None
605
- force : bool
606
- If True, overwrite existing data into destination directories.
607
- If False, raise an error if there are already data into destination directories.
608
- The default is False.
609
- verbose : bool
610
- Whether to print detailed processing information into terminal.
611
- The default is True.
612
- parallel : bool
613
- If True, the files are processed simultaneously in multiple processes.
614
- By default, the number of process is defined with os.cpu_count().
615
- If False, the files are processed sequentially in a single process.
616
- debugging_mode : bool
617
- If True, it reduces the amount of data to process.
618
- For L0A, it processes just the first 3 raw data files.
619
- The default is False.
620
- base_dir : str (optional)
621
- Base directory of DISDRODB. Format: <...>/DISDRODB
622
- If None (the default), the disdrodb config variable 'dir' is used.
864
+ data_source : str
865
+ The name of the institution (for campaigns spanning multiple countries) or
866
+ the name of the country (for campaigns or sensor networks within a single country).
867
+ Must be provided in UPPER CASE.
868
+ campaign_name : str
869
+ The name of the campaign. Must be provided in UPPER CASE.
870
+ station_name : str
871
+ The name of the station.
872
+ force : bool, optional
873
+ If ``True``, existing data in the destination directories will be overwritten.
874
+ If ``False`` (default), an error will be raised if data already exists in the destination directories.
875
+ verbose : bool, optional
876
+ If ``True`` (default), detailed processing information will be printed to the terminal.
877
+ If ``False``, less information will be displayed.
878
+ parallel : bool, optional
879
+ If ``True``, files will be processed in multiple processes simultaneously,
880
+ with each process using a single thread to avoid issues with the HDF/netCDF library.
881
+ If ``False`` (default), files will be processed sequentially in a single process,
882
+ and multi-threading will be automatically exploited to speed up I/O tasks.
883
+ debugging_mode : bool, optional
884
+ If ``True``, the amount of data processed will be reduced.
885
+ Only the first 3 files will be processed. The default value is ``False``.
886
+ remove_l0b: bool, optional
887
+ Whether to remove the processed L0B files. The default value is ``False``.
888
+ data_archive_dir : str, optional
889
+ The base directory of DISDRODB, expected in the format ``<...>/DISDRODB``.
890
+ If not specified, the path specified in the DISDRODB active configuration will be used.
891
+
623
892
  """
624
- run_disdrodb_l0(
625
- base_dir=base_dir,
626
- data_sources=data_sources,
627
- campaign_names=campaign_names,
628
- station_names=station_names,
629
- # L0 archive options
630
- l0a_processing=True,
631
- l0b_processing=False,
632
- l0b_concat=False,
633
- remove_l0a=False,
634
- remove_l0b=False,
635
- # Processing options
636
- force=force,
637
- verbose=verbose,
638
- debugging_mode=debugging_mode,
639
- parallel=parallel,
640
- )
893
+ # Define product
894
+ product = "L0C"
641
895
 
896
+ # Define base directory
897
+ data_archive_dir = get_data_archive_dir(data_archive_dir)
642
898
 
643
- def run_disdrodb_l0b(
644
- data_sources=None,
645
- campaign_names=None,
646
- station_names=None,
647
- # Processing options
648
- force: bool = False,
649
- verbose: bool = False,
650
- debugging_mode: bool = False,
651
- parallel: bool = True,
652
- base_dir: str = None,
653
- remove_l0a: bool = False,
654
- ):
655
- run_disdrodb_l0(
656
- base_dir=base_dir,
657
- data_sources=data_sources,
658
- campaign_names=campaign_names,
659
- station_names=station_names,
660
- # L0 archive options
661
- l0a_processing=False,
662
- l0b_processing=True,
663
- l0b_concat=False,
664
- remove_l0a=remove_l0a,
665
- remove_l0b=False,
666
- # Processing options
667
- force=force,
668
- verbose=verbose,
669
- debugging_mode=debugging_mode,
670
- parallel=parallel,
671
- )
899
+ # Retrieve DISDRODB Metadata Archive directory
900
+ metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
672
901
 
902
+ # ------------------------------------------------------------------------.
903
+ # Start processing
904
+ t_i = time.time()
905
+ msg = f"{product} processing of station {station_name} has started."
906
+ log_info(logger=logger, msg=msg, verbose=verbose)
673
907
 
674
- ####---------------------------------------------------------------------------.
675
- def run_disdrodb_l0b_concat(
676
- data_sources=None,
677
- campaign_names=None,
678
- station_names=None,
679
- remove_l0b=False,
680
- verbose=False,
681
- base_dir=None,
682
- ):
683
- """Concatenate the L0B files of the DISDRODB archive.
908
+ # ------------------------------------------------------------------------.
909
+ # Define logs directory
910
+ logs_dir = create_logs_directory(
911
+ product=product,
912
+ data_archive_dir=data_archive_dir,
913
+ data_source=data_source,
914
+ campaign_name=campaign_name,
915
+ station_name=station_name,
916
+ )
684
917
 
685
- This function is called by the ``disdrodb_run_l0b_concat`` script.
686
- """
687
- from disdrodb.api.io import available_stations
918
+ # Create product directory
919
+ data_dir = create_product_directory(
920
+ data_archive_dir=data_archive_dir,
921
+ metadata_archive_dir=metadata_archive_dir,
922
+ data_source=data_source,
923
+ campaign_name=campaign_name,
924
+ station_name=station_name,
925
+ product=product,
926
+ force=force,
927
+ )
688
928
 
689
- list_info = available_stations(
690
- base_dir=base_dir,
691
- product="L0B",
692
- data_sources=data_sources,
693
- campaign_names=campaign_names,
929
+ # ------------------------------------------------------------------------.
930
+ # Define metadata filepath
931
+ metadata_filepath = define_metadata_filepath(
932
+ metadata_archive_dir=metadata_archive_dir,
933
+ data_source=data_source,
934
+ campaign_name=campaign_name,
935
+ station_name=station_name,
694
936
  )
695
937
 
696
- _check_available_stations(list_info)
697
- list_info = _filter_list_info(list_info, station_names)
938
+ # -------------------------------------------------------------------------.
939
+ # List files to process
940
+ required_product = get_required_product(product)
941
+ flag_not_available_data = False
942
+ try:
943
+ filepaths = find_files(
944
+ data_archive_dir=data_archive_dir,
945
+ data_source=data_source,
946
+ campaign_name=campaign_name,
947
+ station_name=station_name,
948
+ product=required_product,
949
+ # Processing options
950
+ debugging_mode=debugging_mode,
951
+ )
952
+ except Exception as e:
953
+ print(str(e)) # Case where no file paths available
954
+ flag_not_available_data = True
698
955
 
699
- # Print message
700
- n_stations = len(list_info)
701
- print(f"Concatenation of {n_stations} L0B stations started.")
956
+ # -------------------------------------------------------------------------.
957
+ # If no data available, print error message and return None
958
+ if flag_not_available_data:
959
+ msg = (
960
+ f"{product} processing of {data_source} {campaign_name} {station_name}"
961
+ + f"has not been launched because of missing {required_product} data."
962
+ )
963
+ print(msg)
964
+ return
702
965
 
703
- # Start the loop to launch the concatenation of each station
704
- for data_source, campaign_name, station_name in list_info:
705
- print(f"L0B files concatenation of {data_source} {campaign_name} {station_name} station started.")
706
- run_disdrodb_l0b_concat_station(
707
- base_dir=base_dir,
708
- data_source=data_source,
966
+ # -------------------------------------------------------------------------.
967
+ # Retrieve dictionary with the required files for each day.
968
+ dict_days_files = get_files_per_days(filepaths)
969
+
970
+ # -----------------------------------------------------------------.
971
+ # Generate L0C files
972
+ # - Loop over the L0 netCDF files and generate L1 files.
973
+ # - If parallel=True, it does that in parallel using dask.delayed
974
+ list_tasks = [
975
+ _generate_l0c(
976
+ day=day,
977
+ filepaths=filepaths,
978
+ data_dir=data_dir,
979
+ logs_dir=logs_dir,
980
+ metadata_filepath=metadata_filepath,
709
981
  campaign_name=campaign_name,
710
982
  station_name=station_name,
711
- remove_l0b=remove_l0b,
983
+ # Processing options
984
+ force=force,
712
985
  verbose=verbose,
986
+ parallel=parallel,
713
987
  )
714
- print(f"L0 files concatenation of {data_source} {campaign_name} {station_name} station ended.")
988
+ for day, filepaths in dict_days_files.items()
989
+ ]
990
+ list_logs = dask.compute(*list_tasks) if parallel else list_tasks
991
+
992
+ # -----------------------------------------------------------------.
993
+ # Define summary logs
994
+ create_product_logs(
995
+ product=product,
996
+ data_source=data_source,
997
+ campaign_name=campaign_name,
998
+ station_name=station_name,
999
+ data_archive_dir=data_archive_dir,
1000
+ # Logs list
1001
+ list_logs=list_logs,
1002
+ )
715
1003
 
1004
+ # ---------------------------------------------------------------------.
1005
+ # End processing
1006
+ timedelta_str = str(datetime.timedelta(seconds=round(time.time() - t_i)))
1007
+ msg = f"{product} processing of station {station_name} completed in {timedelta_str}"
1008
+ log_info(logger=logger, msg=msg, verbose=verbose)
716
1009
 
717
- ####---------------------------------------------------------------------------.
1010
+ # -----------------------------------------------------------------.
1011
+ # Option to remove L0B
1012
+ if remove_l0b:
1013
+ remove_product(
1014
+ data_archive_dir=data_archive_dir,
1015
+ product="L0B",
1016
+ data_source=data_source,
1017
+ campaign_name=campaign_name,
1018
+ station_name=station_name,
1019
+ logger=logger,
1020
+ verbose=verbose,
1021
+ )