disdrodb 0.1.3__py3-none-any.whl → 0.1.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. disdrodb/__init__.py +4 -0
  2. disdrodb/_version.py +2 -2
  3. disdrodb/api/checks.py +70 -47
  4. disdrodb/api/configs.py +0 -2
  5. disdrodb/api/info.py +3 -3
  6. disdrodb/api/io.py +48 -8
  7. disdrodb/api/path.py +116 -133
  8. disdrodb/api/search.py +12 -3
  9. disdrodb/cli/disdrodb_create_summary.py +103 -0
  10. disdrodb/cli/disdrodb_create_summary_station.py +1 -1
  11. disdrodb/cli/disdrodb_run_l0a_station.py +1 -1
  12. disdrodb/cli/disdrodb_run_l0b_station.py +2 -2
  13. disdrodb/cli/disdrodb_run_l0c_station.py +2 -2
  14. disdrodb/cli/disdrodb_run_l1_station.py +2 -2
  15. disdrodb/cli/disdrodb_run_l2e_station.py +2 -2
  16. disdrodb/cli/disdrodb_run_l2m_station.py +2 -2
  17. disdrodb/data_transfer/download_data.py +123 -7
  18. disdrodb/issue/writer.py +2 -0
  19. disdrodb/l0/l0a_processing.py +10 -5
  20. disdrodb/l0/l0b_nc_processing.py +10 -6
  21. disdrodb/l0/l0b_processing.py +26 -61
  22. disdrodb/l0/l0c_processing.py +369 -251
  23. disdrodb/l0/readers/LPM/ARM/ARM_LPM.py +7 -0
  24. disdrodb/l0/readers/PARSIVEL2/ARM/ARM_PARSIVEL2.py +4 -0
  25. disdrodb/l0/readers/PARSIVEL2/CANADA/UQAM_NC.py +69 -0
  26. disdrodb/l0/readers/PARSIVEL2/MPI/BCO_PARSIVEL2.py +136 -0
  27. disdrodb/l0/readers/PARSIVEL2/MPI/BOWTIE.py +220 -0
  28. disdrodb/l0/readers/PARSIVEL2/NASA/LPVEX.py +109 -0
  29. disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT_NC.py +3 -0
  30. disdrodb/l1/fall_velocity.py +46 -0
  31. disdrodb/l1/processing.py +1 -1
  32. disdrodb/l2/processing.py +1 -1
  33. disdrodb/metadata/checks.py +132 -125
  34. disdrodb/psd/fitting.py +172 -205
  35. disdrodb/psd/models.py +1 -1
  36. disdrodb/routines/__init__.py +54 -0
  37. disdrodb/{l0/routines.py → routines/l0.py} +288 -418
  38. disdrodb/{l1/routines.py → routines/l1.py} +60 -92
  39. disdrodb/{l2/routines.py → routines/l2.py} +249 -462
  40. disdrodb/{routines.py → routines/wrappers.py} +95 -7
  41. disdrodb/scattering/axis_ratio.py +5 -1
  42. disdrodb/scattering/permittivity.py +18 -0
  43. disdrodb/scattering/routines.py +56 -36
  44. disdrodb/summary/routines.py +110 -34
  45. disdrodb/utils/archiving.py +434 -0
  46. disdrodb/utils/cli.py +5 -5
  47. disdrodb/utils/dask.py +62 -1
  48. disdrodb/utils/decorators.py +31 -0
  49. disdrodb/utils/encoding.py +5 -1
  50. disdrodb/{l2 → utils}/event.py +1 -66
  51. disdrodb/utils/logger.py +1 -1
  52. disdrodb/utils/manipulations.py +22 -12
  53. disdrodb/utils/routines.py +166 -0
  54. disdrodb/utils/time.py +3 -291
  55. disdrodb/utils/xarray.py +3 -0
  56. disdrodb/viz/plots.py +85 -14
  57. {disdrodb-0.1.3.dist-info → disdrodb-0.1.4.dist-info}/METADATA +2 -2
  58. {disdrodb-0.1.3.dist-info → disdrodb-0.1.4.dist-info}/RECORD +62 -54
  59. {disdrodb-0.1.3.dist-info → disdrodb-0.1.4.dist-info}/entry_points.txt +1 -0
  60. {disdrodb-0.1.3.dist-info → disdrodb-0.1.4.dist-info}/WHEEL +0 -0
  61. {disdrodb-0.1.3.dist-info → disdrodb-0.1.4.dist-info}/licenses/LICENSE +0 -0
  62. {disdrodb-0.1.3.dist-info → disdrodb-0.1.4.dist-info}/top_level.txt +0 -0
disdrodb/api/path.py CHANGED
@@ -63,6 +63,7 @@ def define_disdrodb_path(
63
63
  The campaign name.
64
64
  check_exists : bool, optional
65
65
  Whether to check if the directory exists. The default value is ``True``.
66
+ Raise error if the directory does not exist.
66
67
 
67
68
  Returns
68
69
  -------
@@ -81,7 +82,7 @@ def define_disdrodb_path(
81
82
  dir_path = os.path.join(archive_dir, ARCHIVE_VERSION, data_source, campaign_name)
82
83
  if check_exists:
83
84
  check_directory_exists(dir_path)
84
- return dir_path
85
+ return os.path.normpath(dir_path)
85
86
 
86
87
 
87
88
  def define_data_source_dir(
@@ -107,6 +108,7 @@ def define_data_source_dir(
107
108
  If not specified, the path specified in the DISDRODB active configuration will be used.
108
109
  check_exists : bool, optional
109
110
  Whether to check if the directory exists. The default value is ``False``.
111
+ Raise error if the directory does not exist.
110
112
 
111
113
  Returns
112
114
  -------
@@ -386,7 +388,7 @@ def define_partitioning_tree(time, folder_partitioning):
386
388
  return os.path.join(year, month, day)
387
389
  if folder_partitioning == "year/month_name":
388
390
  year = str(time.year)
389
- month = str(time.month_name())
391
+ month = time.strftime("%B")
390
392
  return os.path.join(year, month)
391
393
  if folder_partitioning == "year/quarter":
392
394
  year = str(time.year)
@@ -397,7 +399,7 @@ def define_partitioning_tree(time, folder_partitioning):
397
399
  raise NotImplementedError(f"Unrecognized '{folder_partitioning}' folder partitioning scheme.")
398
400
 
399
401
 
400
- def define_file_folder_path(obj, data_dir, folder_partitioning):
402
+ def define_file_folder_path(obj, dir_path, folder_partitioning):
401
403
  """
402
404
  Define the folder path where saving a file based on the dataset's starting time.
403
405
 
@@ -405,12 +407,13 @@ def define_file_folder_path(obj, data_dir, folder_partitioning):
405
407
  ----------
406
408
  ds : xarray.Dataset or pandas.DataFrame
407
409
  The object containing time information.
408
- data_dir : str
410
+ dir : str
409
411
  Directory within the DISDRODB Data Archive where DISDRODB product files are to be saved.
412
+ It can be a product directory or a logs directory.
410
413
  folder_partitioning : str or None
411
414
  Define the subdirectory structure where saving files.
412
415
  Allowed values are:
413
- - None: Files are saved directly in data_dir.
416
+ - None or "": Files are saved directly in data_dir.
414
417
  - "year": Files are saved under a subdirectory for the year.
415
418
  - "year/month": Files are saved under subdirectories for year and month.
416
419
  - "year/month/day": Files are saved under subdirectories for year, month and day
@@ -432,7 +435,7 @@ def define_file_folder_path(obj, data_dir, folder_partitioning):
432
435
 
433
436
  # Build the folder path based on the chosen partition scheme
434
437
  partitioning_tree = define_partitioning_tree(time=starting_time, folder_partitioning=folder_partitioning)
435
- return os.path.join(data_dir, partitioning_tree)
438
+ return os.path.normpath(os.path.join(dir_path, partitioning_tree))
436
439
 
437
440
 
438
441
  def define_product_dir_tree(
@@ -475,15 +478,14 @@ def define_product_dir_tree(
475
478
  check_sample_interval(sample_interval)
476
479
  temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
477
480
  return os.path.join(temporal_resolution)
478
- if product == "L2M":
479
- rolling = product_kwargs.get("rolling")
480
- sample_interval = product_kwargs.get("sample_interval")
481
- model_name = product_kwargs.get("model_name")
482
- check_rolling(rolling)
483
- check_sample_interval(sample_interval)
484
- temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
485
- return os.path.join(model_name, temporal_resolution)
486
- raise ValueError(f"The product {product} is not defined.")
481
+ # L2M if product == "L2M":
482
+ rolling = product_kwargs.get("rolling")
483
+ sample_interval = product_kwargs.get("sample_interval")
484
+ model_name = product_kwargs.get("model_name")
485
+ check_rolling(rolling)
486
+ check_sample_interval(sample_interval)
487
+ temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
488
+ return os.path.join(model_name, temporal_resolution)
487
489
 
488
490
 
489
491
  def define_logs_dir(
@@ -529,7 +531,7 @@ def define_logs_dir(
529
531
  product=product,
530
532
  **product_kwargs,
531
533
  )
532
- logs_dir = os.path.join(campaign_dir, "logs", "files", product, product_dir_tree, station_name)
534
+ logs_dir = os.path.normpath(os.path.join(campaign_dir, "logs", "files", product, product_dir_tree, station_name))
533
535
  if check_exists:
534
536
  check_directory_exists(logs_dir)
535
537
  return str(logs_dir)
@@ -643,7 +645,7 @@ def define_data_dir(
643
645
  **product_kwargs,
644
646
  )
645
647
  # Define data directory
646
- data_dir = os.path.join(station_dir, product_dir_tree)
648
+ data_dir = os.path.normpath(os.path.join(station_dir, product_dir_tree))
647
649
  # Check if directory exists
648
650
  if check_exists:
649
651
  check_directory_exists(data_dir)
@@ -674,7 +676,8 @@ def define_filename(
674
676
  campaign_name: str,
675
677
  station_name: str,
676
678
  # Filename options
677
- obj=None,
679
+ start_time=None,
680
+ end_time=None,
678
681
  add_version=True,
679
682
  add_time_period=True,
680
683
  add_extension=True,
@@ -688,19 +691,22 @@ def define_filename(
688
691
 
689
692
  Parameters
690
693
  ----------
691
- obj : xarray.Dataset or pandas.DataFrame
692
- xarray Dataset or pandas DataFrame.
693
- Required if add_time_period = True.
694
694
  campaign_name : str
695
695
  Name of the campaign.
696
696
  station_name : str
697
697
  Name of the station.
698
+ start_time : datetime.datatime, optional
699
+ Start time.
700
+ Required if add_time_period = True.
701
+ end_time : datetime.datatime, optional
702
+ End time.
703
+ Required if add_time_period = True.
698
704
  sample_interval : int, optional
699
705
  The sampling interval in seconds of the product.
700
- It must be specified only for product L2E and L2M !
706
+ It must be specified only for product L0C, L1, L2E and L2M !
701
707
  rolling : bool, optional
702
708
  Whether the dataset has been resampled by aggregating or rolling.
703
- It must be specified only for product L2E and L2M !
709
+ It must be specified only for product L1, L2E and L2M !
704
710
  model_name : str
705
711
  The model name of the fitted statistical distribution for the DSD.
706
712
  It must be specified only for product L2M !
@@ -715,19 +721,25 @@ def define_filename(
715
721
  product = check_product(product)
716
722
  product_kwargs = check_product_kwargs(product, product_kwargs)
717
723
 
718
- # -----------------------------------------.
719
- # TODO: Define temporal_resolution
720
- # - ADD temporal_resolution also to L0A and L0B
721
- # - Add temporal_resolution also to L0C and L1
724
+ if add_time_period and (start_time is None or end_time is None):
725
+ raise ValueError("If add_time_period=True, specify start_time and end_time.")
722
726
 
723
727
  # -----------------------------------------.
724
728
  # Define product name
725
729
  product_name = f"{product}"
730
+
731
+ # L0C ... sample interval known only per-file
732
+ # L1 ... in future known a priori
733
+ # if product in ["L1"]:
734
+ # # TODO: HACK FOR CURRENT L0C and L1 log files in create_product_logs
735
+ # sample_interval = product_kwargs.get("sample_interval", 0)
736
+ # temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=False)
737
+ # product_name = f"{product}.{temporal_resolution}"
726
738
  if product in ["L2E", "L2M"]:
727
739
  rolling = product_kwargs.get("rolling")
728
740
  sample_interval = product_kwargs.get("sample_interval")
729
741
  temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
730
- product_name = f"L2E.{temporal_resolution}"
742
+ product_name = f"{product}.{temporal_resolution}"
731
743
  if product in ["L2M"]:
732
744
  model_name = product_kwargs.get("model_name")
733
745
  product_name = f"L2M_{model_name}.{temporal_resolution}"
@@ -744,10 +756,9 @@ def define_filename(
744
756
  # -----------------------------------------.
745
757
  # Add time period information
746
758
  if add_time_period:
747
- starting_time, ending_time = get_file_start_end_time(obj)
748
- starting_time = starting_time.strftime("%Y%m%d%H%M%S")
749
- ending_time = ending_time.strftime("%Y%m%d%H%M%S")
750
- filename = f"{filename}.s{starting_time}.e{ending_time}"
759
+ start_time = start_time.strftime("%Y%m%d%H%M%S")
760
+ end_time = end_time.strftime("%Y%m%d%H%M%S")
761
+ filename = f"{filename}.s{start_time}.e{end_time}"
751
762
 
752
763
  # -----------------------------------------.
753
764
  # Add product version
@@ -784,56 +795,40 @@ def define_l0a_filename(df, campaign_name: str, station_name: str) -> str:
784
795
  L0A file name.
785
796
  """
786
797
  starting_time, ending_time = get_file_start_end_time(df)
787
- starting_time = starting_time.strftime("%Y%m%d%H%M%S")
788
- ending_time = ending_time.strftime("%Y%m%d%H%M%S")
789
- version = ARCHIVE_VERSION
790
- filename = f"L0A.{campaign_name}.{station_name}.s{starting_time}.e{ending_time}.{version}.parquet"
798
+ filename = define_filename(
799
+ product="L0A",
800
+ campaign_name=campaign_name,
801
+ station_name=station_name,
802
+ # Filename options
803
+ start_time=starting_time,
804
+ end_time=ending_time,
805
+ add_version=True,
806
+ add_time_period=True,
807
+ add_extension=True,
808
+ )
791
809
  return filename
792
810
 
793
811
 
794
812
  def define_l0b_filename(ds, campaign_name: str, station_name: str) -> str:
795
- """Define L0B file name.
796
-
797
- Parameters
798
- ----------
799
- ds : xarray.Dataset
800
- L0B xarray Dataset.
801
- campaign_name : str
802
- Name of the campaign.
803
- station_name : str
804
- Name of the station.
805
-
806
- Returns
807
- -------
808
- str
809
- L0B file name.
810
- """
813
+ """Define L0B file name."""
811
814
  starting_time, ending_time = get_file_start_end_time(ds)
812
- starting_time = starting_time.strftime("%Y%m%d%H%M%S")
813
- ending_time = ending_time.strftime("%Y%m%d%H%M%S")
814
- version = ARCHIVE_VERSION
815
- filename = f"L0B.{campaign_name}.{station_name}.s{starting_time}.e{ending_time}.{version}.nc"
815
+ filename = define_filename(
816
+ product="L0B",
817
+ campaign_name=campaign_name,
818
+ station_name=station_name,
819
+ # Filename options
820
+ start_time=starting_time,
821
+ end_time=ending_time,
822
+ add_version=True,
823
+ add_time_period=True,
824
+ add_extension=True,
825
+ )
816
826
  return filename
817
827
 
818
828
 
819
829
  def define_l0c_filename(ds, campaign_name: str, station_name: str) -> str:
820
- """Define L0C file name.
821
-
822
- Parameters
823
- ----------
824
- ds : xarray.Dataset
825
- L0B xarray Dataset.
826
- campaign_name : str
827
- Name of the campaign.
828
- station_name : str
829
- Name of the station.
830
-
831
- Returns
832
- -------
833
- str
834
- L0B file name.
835
- """
836
- # TODO: add sample_interval as argument
830
+ """Define L0C file name."""
831
+ # TODO: add sample_interval as function argument s
837
832
  sample_interval = int(ensure_sample_interval_in_seconds(ds["sample_interval"]).data.item())
838
833
  temporal_resolution = define_temporal_resolution(sample_interval, rolling=False)
839
834
  starting_time, ending_time = get_file_start_end_time(ds)
@@ -845,23 +840,10 @@ def define_l0c_filename(ds, campaign_name: str, station_name: str) -> str:
845
840
 
846
841
 
847
842
  def define_l1_filename(ds, campaign_name, station_name: str) -> str:
848
- """Define L1 file name.
843
+ """Define L1 file name."""
844
+ # TODO: add sample_interval and rolling as function argument
849
845
 
850
- Parameters
851
- ----------
852
- ds : xarray.Dataset
853
- L1 xarray Dataset.
854
- campaign_name : str
855
- Name of the campaign.
856
- station_name : str
857
- Name of the station.
858
-
859
- Returns
860
- -------
861
- str
862
- L1 file name.
863
- """
864
- # TODO: add sample_interval as argument
846
+ starting_time, ending_time = get_file_start_end_time(ds)
865
847
  sample_interval = int(ensure_sample_interval_in_seconds(ds["sample_interval"]).data.item())
866
848
  temporal_resolution = define_temporal_resolution(sample_interval, rolling=False)
867
849
  starting_time, ending_time = get_file_start_end_time(ds)
@@ -869,32 +851,41 @@ def define_l1_filename(ds, campaign_name, station_name: str) -> str:
869
851
  ending_time = ending_time.strftime("%Y%m%d%H%M%S")
870
852
  version = ARCHIVE_VERSION
871
853
  filename = f"L1.{temporal_resolution}.{campaign_name}.{station_name}.s{starting_time}.e{ending_time}.{version}.nc"
854
+
855
+ # filename = define_filename(
856
+ # product="L1",
857
+ # campaign_name=campaign_name,
858
+ # station_name=station_name,
859
+ # # Filename options
860
+ # start_time=starting_time,
861
+ # end_time=ending_time,
862
+ # add_version=True,
863
+ # add_time_period=True,
864
+ # add_extension=True,
865
+ # # Product options
866
+ # # sample_interval=sample_interval,
867
+ # # rolling=rolling,
868
+ # )
872
869
  return filename
873
870
 
874
871
 
875
872
  def define_l2e_filename(ds, campaign_name: str, station_name: str, sample_interval: int, rolling: bool) -> str:
876
- """Define L2E file name.
877
-
878
- Parameters
879
- ----------
880
- ds : xarray.Dataset
881
- L1 xarray Dataset
882
- campaign_name : str
883
- Name of the campaign.
884
- station_name : str
885
- Name of the station
886
-
887
- Returns
888
- -------
889
- str
890
- L0B file name.
891
- """
892
- temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
873
+ """Define L2E file name."""
893
874
  starting_time, ending_time = get_file_start_end_time(ds)
894
- starting_time = starting_time.strftime("%Y%m%d%H%M%S")
895
- ending_time = ending_time.strftime("%Y%m%d%H%M%S")
896
- version = ARCHIVE_VERSION
897
- filename = f"L2E.{temporal_resolution}.{campaign_name}.{station_name}.s{starting_time}.e{ending_time}.{version}.nc"
875
+ filename = define_filename(
876
+ product="L2E",
877
+ campaign_name=campaign_name,
878
+ station_name=station_name,
879
+ # Filename options
880
+ start_time=starting_time,
881
+ end_time=ending_time,
882
+ add_version=True,
883
+ add_time_period=True,
884
+ add_extension=True,
885
+ # Product options
886
+ sample_interval=sample_interval,
887
+ rolling=rolling,
888
+ )
898
889
  return filename
899
890
 
900
891
 
@@ -906,29 +897,21 @@ def define_l2m_filename(
906
897
  rolling: bool,
907
898
  model_name: str,
908
899
  ) -> str:
909
- """Define L2M file name.
910
-
911
- Parameters
912
- ----------
913
- ds : xarray.Dataset
914
- L1 xarray Dataset
915
- campaign_name : str
916
- Name of the campaign.
917
- station_name : str
918
- Name of the station
919
-
920
- Returns
921
- -------
922
- str
923
- L0B file name.
924
- """
925
- temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
900
+ """Define L2M file name."""
926
901
  starting_time, ending_time = get_file_start_end_time(ds)
927
- starting_time = starting_time.strftime("%Y%m%d%H%M%S")
928
- ending_time = ending_time.strftime("%Y%m%d%H%M%S")
929
- version = ARCHIVE_VERSION
930
- filename = (
931
- f"L2M_{model_name}.{temporal_resolution}.{campaign_name}."
932
- + f"{station_name}.s{starting_time}.e{ending_time}.{version}.nc"
902
+ filename = define_filename(
903
+ product="L2M",
904
+ campaign_name=campaign_name,
905
+ station_name=station_name,
906
+ # Filename options
907
+ start_time=starting_time,
908
+ end_time=ending_time,
909
+ add_version=True,
910
+ add_time_period=True,
911
+ add_extension=True,
912
+ # Product options
913
+ sample_interval=sample_interval,
914
+ rolling=rolling,
915
+ model_name=model_name,
933
916
  )
934
917
  return filename
disdrodb/api/search.py CHANGED
@@ -20,6 +20,8 @@ from disdrodb.constants import PRODUCTS_REQUIREMENTS
20
20
  from disdrodb.utils.directories import contains_files, contains_netcdf_or_parquet_files, list_directories, list_files
21
21
  from disdrodb.utils.yaml import read_yaml
22
22
 
23
+ ####-------------------------------------------------------------------------
24
+
23
25
 
24
26
  def get_required_product(product):
25
27
  """Determine the required product for input product processing."""
@@ -31,7 +33,7 @@ def get_required_product(product):
31
33
 
32
34
 
33
35
  ####-------------------------------------------------------------------------
34
- #### List DISDRODB infrastructure directories
36
+ #### List DISDRODB Metadata directories
35
37
 
36
38
 
37
39
  def list_data_sources(metadata_archive_dir, data_sources=None, invalid_fields_policy="raise"):
@@ -167,6 +169,10 @@ def list_station_names(
167
169
  return station_names
168
170
 
169
171
 
172
+ ####-------------------------------------------------------------------------
173
+ #### Filtering utilities for available_stations
174
+
175
+
170
176
  def _finalize_output(list_info, return_tuple):
171
177
  # - Return the (data_source, campaign_name, station_name) tuple
172
178
  if return_tuple:
@@ -228,7 +234,6 @@ def keep_list_info_elements_with_product_data(data_archive_dir, product, list_in
228
234
  checking_function = contains_files if product == "RAW" else contains_netcdf_or_parquet_files
229
235
 
230
236
  # Check presence of data for each station
231
- # TODO: - In parallel over stations to speed up ?
232
237
  list_info_with_product_data = []
233
238
  for data_source, campaign_name, station_name in list_info:
234
239
  data_dir = define_data_dir(
@@ -362,10 +367,11 @@ def available_stations(
362
367
  metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
363
368
  product = check_product(product) if product is not None else None
364
369
  invalid_fields_policy = check_invalid_fields_policy(invalid_fields_policy)
370
+
365
371
  # Retrieve available stations from the Metadata Archive
366
372
  # - Raise error if no stations availables !
367
373
  list_info = list_station_names(
368
- metadata_archive_dir,
374
+ metadata_archive_dir=metadata_archive_dir,
369
375
  data_sources=data_sources,
370
376
  campaign_names=campaign_names,
371
377
  station_names=station_names,
@@ -484,3 +490,6 @@ def available_campaigns(
484
490
  campaign_names = [info[1] for info in list_info]
485
491
  campaign_names = np.unique(campaign_names).tolist()
486
492
  return campaign_names
493
+
494
+
495
+ ####-------------------------------------------------------------------------
@@ -0,0 +1,103 @@
1
+ # -----------------------------------------------------------------------------.
2
+ # Copyright (c) 2021-2023 DISDRODB developers
3
+ #
4
+ # This program is free software: you can redistribute it and/or modify
5
+ # it under the terms of the GNU General Public License as published by
6
+ # the Free Software Foundation, either version 3 of the License, or
7
+ # (at your option) any later version.
8
+ #
9
+ # This program is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
+ # GNU General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU General Public License
15
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
16
+ # -----------------------------------------------------------------------------.
17
+ """Script to create summary figures and tables for a DISDRODB stationn."""
18
+ import sys
19
+ from typing import Optional
20
+
21
+ import click
22
+
23
+ from disdrodb.utils.cli import (
24
+ click_data_archive_dir_option,
25
+ click_metadata_archive_dir_option,
26
+ click_stations_options,
27
+ parse_archive_dir,
28
+ parse_arg_to_list,
29
+ )
30
+
31
+ sys.tracebacklimit = 0 # avoid full traceback error if occur
32
+
33
+ # -------------------------------------------------------------------------.
34
+ # Click Command Line Interface decorator
35
+
36
+
37
+ @click.command()
38
+ @click_stations_options
39
+ @click_data_archive_dir_option
40
+ @click_metadata_archive_dir_option
41
+ @click.option("-p", "--parallel", type=bool, show_default=True, default=False, help="Read files in parallel")
42
+ def disdrodb_create_summary(
43
+ # Stations options
44
+ data_sources: Optional[str] = None,
45
+ campaign_names: Optional[str] = None,
46
+ station_names: Optional[str] = None,
47
+ # Processing options:
48
+ parallel=False,
49
+ # DISDRODB root directories
50
+ data_archive_dir: Optional[str] = None,
51
+ metadata_archive_dir: Optional[str] = None,
52
+ ):
53
+ r"""Create summary figures and tables for a specific set of DISDRODB stations.
54
+
55
+ Parameters \n
56
+ ---------- \n
57
+ data_sources : str
58
+ Name of data source(s) to process.
59
+ The name(s) must be UPPER CASE.
60
+ If campaign_names and station are not specified, process all stations.
61
+ To specify multiple data sources, write i.e.: --data_sources 'GPM EPFL NCAR'
62
+ campaign_names : str
63
+ Name of the campaign(s) for which to create stations summaries.
64
+ The name(s) must be UPPER CASE.
65
+ To specify multiple campaigns, write i.e.: --campaign_names 'IPEX IMPACTS'
66
+ station_names : str
67
+ Station names.
68
+ To specify multiple stations, write i.e.: --station_names 'station1 station2'
69
+ data_archive_dir : str \n
70
+ DISDRODB Data Archive directory \n
71
+ Format: <...>/DISDRODB \n
72
+ If not specified, uses path specified in the DISDRODB active configuration. \n
73
+ """
74
+ from disdrodb.routines import create_summary
75
+ from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
76
+
77
+ data_archive_dir = parse_archive_dir(data_archive_dir)
78
+ data_sources = parse_arg_to_list(data_sources)
79
+ campaign_names = parse_arg_to_list(campaign_names)
80
+ station_names = parse_arg_to_list(station_names)
81
+
82
+ # -------------------------------------------------------------------------.
83
+ # If parallel=True, set the dask environment
84
+ if parallel:
85
+ cluster, client = initialize_dask_cluster()
86
+
87
+ # -------------------------------------------------------------------------.
88
+ create_summary(
89
+ # Station arguments
90
+ data_sources=data_sources,
91
+ campaign_names=campaign_names,
92
+ station_names=station_names,
93
+ # Options
94
+ parallel=parallel,
95
+ # DISDRODB root directory
96
+ data_archive_dir=data_archive_dir,
97
+ metadata_archive_dir=metadata_archive_dir,
98
+ )
99
+
100
+ # -------------------------------------------------------------------------.
101
+ # Close the cluster
102
+ if parallel:
103
+ close_dask_cluster(cluster, client)
@@ -14,7 +14,7 @@
14
14
  # You should have received a copy of the GNU General Public License
15
15
  # along with this program. If not, see <http://www.gnu.org/licenses/>.
16
16
  # -----------------------------------------------------------------------------.
17
- """Script to run the DISDRODB L0 station processing."""
17
+ """Script to create summary figures and tables for a DISDRODB station."""
18
18
  import sys
19
19
  from typing import Optional
20
20
 
@@ -89,7 +89,7 @@ def disdrodb_run_l0a_station(
89
89
  Format: <...>/DISDRODB
90
90
  If not specified, uses path specified in the DISDRODB active configuration.
91
91
  """
92
- from disdrodb.l0.routines import run_l0a_station
92
+ from disdrodb.routines.l0 import run_l0a_station
93
93
  from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
94
94
 
95
95
  data_archive_dir = parse_archive_dir(data_archive_dir)
@@ -91,7 +91,7 @@ def disdrodb_run_l0b_station(
91
91
  Format: <...>/DISDRODB
92
92
  If not specified, uses path specified in the DISDRODB active configuration.
93
93
  """
94
- from disdrodb.l0.routines import run_l0b_station
94
+ from disdrodb.routines.l0 import run_l0b_station
95
95
  from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
96
96
 
97
97
  data_archive_dir = parse_archive_dir(data_archive_dir)
@@ -100,7 +100,7 @@ def disdrodb_run_l0b_station(
100
100
  # -------------------------------------------------------------------------.
101
101
  # If parallel=True, set the dask environment
102
102
  if parallel:
103
- cluster, client = initialize_dask_cluster()
103
+ cluster, client = initialize_dask_cluster(minimum_memory="4GB")
104
104
 
105
105
  # -------------------------------------------------------------------------.
106
106
  run_l0b_station(
@@ -94,7 +94,7 @@ def disdrodb_run_l0c_station(
94
94
  Format: <...>/DISDRODB
95
95
  If not specified, uses path specified in the DISDRODB active configuration.
96
96
  """
97
- from disdrodb.l0.routines import run_l0c_station
97
+ from disdrodb.routines.l0 import run_l0c_station
98
98
  from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
99
99
 
100
100
  data_archive_dir = parse_archive_dir(data_archive_dir)
@@ -103,7 +103,7 @@ def disdrodb_run_l0c_station(
103
103
  # -------------------------------------------------------------------------.
104
104
  # If parallel=True, set the dask environment
105
105
  if parallel:
106
- cluster, client = initialize_dask_cluster()
106
+ cluster, client = initialize_dask_cluster(minimum_memory="4GB")
107
107
 
108
108
  # -------------------------------------------------------------------------.
109
109
  run_l0c_station(
@@ -89,7 +89,7 @@ def disdrodb_run_l1_station(
89
89
  Format: <...>/DISDRODB
90
90
  If not specified, uses path specified in the DISDRODB active configuration.
91
91
  """
92
- from disdrodb.l1.routines import run_l1_station
92
+ from disdrodb.routines.l1 import run_l1_station
93
93
  from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
94
94
 
95
95
  data_archive_dir = parse_archive_dir(data_archive_dir)
@@ -97,7 +97,7 @@ def disdrodb_run_l1_station(
97
97
  # -------------------------------------------------------------------------.
98
98
  # If parallel=True, set the dask environment
99
99
  if parallel:
100
- cluster, client = initialize_dask_cluster()
100
+ cluster, client = initialize_dask_cluster(minimum_memory="4GB")
101
101
 
102
102
  # -------------------------------------------------------------------------.
103
103
  run_l1_station(
@@ -89,7 +89,7 @@ def disdrodb_run_l2e_station(
89
89
  Format: <...>/DISDRODB
90
90
  If not specified, uses path specified in the DISDRODB active configuration.
91
91
  """
92
- from disdrodb.l2.routines import run_l2e_station
92
+ from disdrodb.routines.l2 import run_l2e_station
93
93
  from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
94
94
 
95
95
  data_archive_dir = parse_archive_dir(data_archive_dir)
@@ -98,7 +98,7 @@ def disdrodb_run_l2e_station(
98
98
  # -------------------------------------------------------------------------.
99
99
  # If parallel=True, set the dask environment
100
100
  if parallel:
101
- cluster, client = initialize_dask_cluster(minimum_memory="8GB")
101
+ cluster, client = initialize_dask_cluster(minimum_memory="4GB")
102
102
 
103
103
  # -------------------------------------------------------------------------.
104
104
  run_l2e_station(
@@ -89,7 +89,7 @@ def disdrodb_run_l2m_station(
89
89
  Format: <...>/DISDRODB
90
90
  If not specified, uses path specified in the DISDRODB active configuration.
91
91
  """
92
- from disdrodb.l2.routines import run_l2m_station
92
+ from disdrodb.routines.l2 import run_l2m_station
93
93
  from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
94
94
 
95
95
  data_archive_dir = parse_archive_dir(data_archive_dir)
@@ -98,7 +98,7 @@ def disdrodb_run_l2m_station(
98
98
  # -------------------------------------------------------------------------.
99
99
  # If parallel=True, set the dask environment
100
100
  if parallel:
101
- cluster, client = initialize_dask_cluster()
101
+ cluster, client = initialize_dask_cluster(minimum_memory="4GB")
102
102
 
103
103
  # -------------------------------------------------------------------------.
104
104
  run_l2m_station(