disdrodb 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

disdrodb/cli/disdrodb_run_l2e_station.py

@@ -89,7 +89,7 @@ def disdrodb_run_l2e_station(
         Format: <...>/DISDRODB
         If not specified, uses path specified in the DISDRODB active configuration.
     """
-    from disdrodb.l2.routines import run_l2e_station
+    from disdrodb.routines.l2 import run_l2e_station
     from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
 
     data_archive_dir = parse_archive_dir(data_archive_dir)
@@ -98,7 +98,7 @@ def disdrodb_run_l2e_station(
     # -------------------------------------------------------------------------.
     # If parallel=True, set the dask environment
     if parallel:
-        cluster, client = initialize_dask_cluster(minimum_memory="8GB")
+        cluster, client = initialize_dask_cluster(minimum_memory="4GB")
 
     # -------------------------------------------------------------------------.
     run_l2e_station(

disdrodb/cli/disdrodb_run_l2m_station.py

@@ -89,7 +89,7 @@ def disdrodb_run_l2m_station(
         Format: <...>/DISDRODB
         If not specified, uses path specified in the DISDRODB active configuration.
     """
-    from disdrodb.l2.routines import run_l2m_station
+    from disdrodb.routines.l2 import run_l2m_station
     from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
 
     data_archive_dir = parse_archive_dir(data_archive_dir)
@@ -98,7 +98,7 @@ def disdrodb_run_l2m_station(
     # -------------------------------------------------------------------------.
     # If parallel=True, set the dask environment
     if parallel:
-        cluster, client = initialize_dask_cluster()
+        cluster, client = initialize_dask_cluster(minimum_memory="4GB")
 
     # -------------------------------------------------------------------------.
     run_l2m_station(

disdrodb/constants.py

@@ -41,7 +41,7 @@ COORDINATES = [
     "time",
     "sample_interval",
 ]
-OPTICAL_SENSORS = ["PARSIVEL", "PARSIVEL2", "LPM", "PWS100"]
+OPTICAL_SENSORS = ["PARSIVEL", "PARSIVEL2", "LPM", "PWS100", "SWS250"]
 IMPACT_SENSORS = ["RD80"]
 
 PRODUCTS = ["RAW", "L0A", "L0B", "L0C", "L1", "L2E", "L2M"]

disdrodb/data_transfer/download_data.py

@@ -239,7 +239,7 @@ def check_consistent_station_name(metadata_filepath, station_name):
     return station_name
 
 
-def download_station_data(metadata_filepath: str, data_archive_dir: str, force: bool = False) -> None:
+def download_station_data(metadata_filepath: str, data_archive_dir: str, force: bool = False, verbose=True) -> None:
     """Download and unzip the station data .
 
     Parameters
@@ -275,17 +275,27 @@ def download_station_data(metadata_filepath: str, data_archive_dir: str, force:
         raise ValueError(f"Invalid disdrodb_data_url '{disdrodb_data_url}' for station {station_name}")
 
     # Download files
-    # - Option 1: Download Zip file containing all station raw data
+    # - Option 1: Download ZIP file containing all station raw data
     if disdrodb_data_url.startswith("https://zenodo.org/") or disdrodb_data_url.startswith("https://cloudnet.fmi.fi/"):
         download_zip_file(url=disdrodb_data_url, dst_dir=station_dir, force=force)
+
     # - Option 2: Recursive download from a web server via HTTP or HTTPS.
     elif disdrodb_data_url.startswith("http"):
-        download_web_server_data(url=disdrodb_data_url, dst_dir=station_dir, force=force, verbose=True)
+        download_web_server_data(url=disdrodb_data_url, dst_dir=station_dir, force=force, verbose=verbose)
+        # - Retry to be more sure that all data have been downloaded
+        download_web_server_data(url=disdrodb_data_url, dst_dir=station_dir, force=True, verbose=verbose)
+
+    # - Option 3: Recursive download from a ftp server
+    elif disdrodb_data_url.startswith("ftp"):
+        download_ftp_server_data(url=disdrodb_data_url, dst_dir=station_dir, force=force, verbose=verbose)
+        # - Retry to be more sure that all data have been downloaded
+        download_ftp_server_data(url=disdrodb_data_url, dst_dir=station_dir, force=True, verbose=verbose)
+
     else:
         raise NotImplementedError(f"Open a GitHub Issue to enable the download of data from {disdrodb_data_url}.")
 
 
-####-----------------------------------------------------------------------------------------.
+####--------------------------------------------------------------------.
 #### Download from Web Server via HTTP or HTTPS
 
 
@@ -301,9 +311,17 @@ def download_web_server_data(url: str, dst_dir: str, force=True, verbose=True) -
     3. Compute cut-dirs so that only the last segment of the path remains locally.
     4. Build and run the wget command.
 
-    Example:
-        download_with_wget("https://ruisdael.citg.tudelft.nl/parsivel/PAR001_Cabauw/2021/202101/")
-        # → Creates a local folder "202101/" with all files and subfolders.
+    Parameters
+    ----------
+    url : str
+        HTTPS URL pointing to webserver folder. Example: "https://ruisdael.citg.tudelft.nl/parsivel/PAR001_Cabauw/"
+    dst_dir : str
+         Local directory where to download the file (DISDRODB station data directory).
+    force : bool, optional
+        If ``True``, re-download new/updated files (skip unchanged ones).
+        If ``False``, keep existing files untouched.
+    verbose : bool, optional
+        Print wget output (default is True).
     """
     # 1. Ensure wget exists
     ensure_wget_available()
@@ -393,6 +411,104 @@ def build_webserver_wget_command(url: str, cut_dirs: int, dst_dir: str, force: b
     return cmd
 
 
+####--------------------------------------------------------------------.
+#### Download from FTP Server
+
+
+def build_ftp_server_wget_command(
+    url: str,
+    cut_dirs: int,
+    dst_dir: str,
+    force: bool,
+    verbose: bool,
+) -> list[str]:
+    """Construct the wget command list for FTP recursive download.
+
+    Parameters
+    ----------
+    url : str
+        FTP URL to download from.
+    cut_dirs : int
+        Number of leading path components to strip.
+    dst_dir : str
+        Local destination directory.
+    force : bool
+        If True, re-download newer files (--timestamping).
+        If False, keep existing files untouched (--no-clobber).
+    verbose : bool
+        If False, suppress wget output (-q).
+    """
+    cmd = ["wget"]  # base command
+
+    if not verbose:
+        cmd.append("-q")  # quiet mode --> no output except errors
+
+    cmd += [
+        "-r",  # recursive --> traverse into subdirectories
+        "-np",  # no parent --> don't ascend to higher-level dirs
+        "-nH",  # no host dirs --> avoid creating ftp.example.com/ locally
+        f"--cut-dirs={cut_dirs}",  # strip N leading path components
+    ]
+
+    if force:
+        cmd.append("--timestamping")  # download if remote file is newer
+    else:
+        cmd.append("--no-clobber")  # skip files that already exist
+
+    cmd += [
+        "-P",  # specify local destination directory
+        dst_dir,
+        f"ftp://anonymous:disdrodb@{url}",  # target FTP URL
+    ]
+    return cmd
+
+
+def download_ftp_server_data(url: str, dst_dir: str, force: bool = False, verbose: bool = True) -> None:
+    """Download data from an FTP server with anonymous login.
+
+    Parameters
+    ----------
+    url : str
+        FTP server URL pointing to a folder. Example: "ftp://ftp.example.com/path/to/data/"
+    dst_dir : str
+         Local directory where to download the file (DISDRODB station data directory).
+    force : bool, optional
+        If ``True``, re-download new/updated files (skip unchanged ones).
+        If ``False``, keep existing files untouched.
+    verbose : bool, optional
+        Print wget output (default is True).
+    """
+    ensure_wget_available()
+
+    # Ensure trailing slash
+    url = ensure_trailing_slash(url)
+
+    # Compute cut-dirs so files land directly in dst_dir
+    cut_dirs = compute_cut_dirs(url)
+
+    # Make destination directory
+    os.makedirs(dst_dir, exist_ok=True)
+
+    # Build wget command
+    cmd = build_ftp_server_wget_command(
+        url,
+        cut_dirs=cut_dirs,
+        dst_dir=dst_dir,
+        force=force,
+        verbose=verbose,
+    )
+    # Run wget
+    try:
+        subprocess.run(cmd, check=True)
+    except subprocess.CalledProcessError as e:
+        raise subprocess.CalledProcessError(
+            returncode=e.returncode,
+            cmd=e.cmd,
+            output=e.output,
+            stderr=e.stderr,
+        )
+
+
 ####--------------------------------------------------------------------.
 #### Download from Zenodo
 

disdrodb/etc/products/L1/global.yaml

@@ -1,5 +1,5 @@
 product_options:
-  fall_velocity_method: "Beard1976"
+  fall_velocity_model: "Beard1976"
   minimum_diameter: 0
   maximum_diameter: 10
   minimum_velocity: 0

disdrodb/etc/products/L2E/5MIN.yaml

@@ -0,0 +1 @@
+radar_enabled: True

disdrodb/etc/products/L2E/global.yaml

@@ -1,4 +1,4 @@
-temporal_resolutions: ["1MIN", "5MIN", "10MIN", "ROLL1MIN"]
+temporal_resolutions: [1MIN"] # "5MIN", "10MIN", "ROLL1MIN"]
 archive_options:
   strategy: time_block
   strategy_options:

disdrodb/etc/products/L2M/GAMMA_GS_ND_MAE.yaml

@@ -0,0 +1,6 @@
+psd_model: "GammaPSD"
+optimization: "GS"
+optimization_kwargs:
+  target: "ND"
+  transformation: "identity"
+  error_order: 1

disdrodb/etc/products/L2M/GAMMA_ML.yaml

@@ -1,7 +1,7 @@
 psd_model: "GammaPSD"
 optimization: "ML"
 optimization_kwargs:
-  init_method: "M346"
+  init_method: "None"
   probability_method: "cdf"
   likelihood: "multinomial"
   truncated_likelihood: True

disdrodb/etc/products/L2M/LOGNORMAL_GS_LOG_ND_MAE.yaml

@@ -0,0 +1,6 @@
+psd_model: "LognormalPSD"
+optimization: "GS"
+optimization_kwargs:
+  target: "ND"
+  transformation: "log"
+  error_order: 1

disdrodb/etc/products/L2M/LOGNORMAL_GS_ND_MAE.yaml

@@ -0,0 +1,6 @@
+psd_model: "LognormalPSD"
+optimization: "GS"
+optimization_kwargs:
+  target: "ND"
+  transformation: "identity"
+  error_order: 1

disdrodb/etc/products/L2M/LOGNORMAL_ML.yaml

@@ -0,0 +1,8 @@
+psd_model: "LognormalPSD"
+optimization: "ML"
+optimization_kwargs:
+  init_method: "None"
+  probability_method: "cdf"
+  likelihood: "multinomial"
+  truncated_likelihood: True
+  optimizer: "Nelder-Mead"

disdrodb/etc/products/L2M/global.yaml

@@ -1,12 +1,20 @@
-temporal_resolutions: ["1MIN", "5MIN", "10MIN"]
-models: ["GAMMA_ML", "NGAMMA_GS_LOG_ND_MAE"]
+temporal_resolutions: ["1MIN"] #, "5MIN", "10MIN"]
+models:
+  [
+    "GAMMA_ML",
+    "GAMMA_GS_ND_MAE",
+    "NGAMMA_GS_LOG_ND_MAE",
+    "NGAMMA_GS_ND_MAE",
+    "LOGNORMAL_ML",
+    "LOGNORMAL_GS_ND_MAE",
+  ]
 archive_options:
   strategy: time_block
   strategy_options:
     freq: month
   folder_partitioning: ""
 product_options:
-  fall_velocity_method: "Beard1976"
+  fall_velocity_model: "Beard1976"
   diameter_min: 0
   diameter_max: 10
   diameter_spacing: 0.05

disdrodb/issue/writer.py

@@ -120,9 +120,11 @@ def create_station_issue(data_source, campaign_name, station_name, metadata_arch
     )
     if os.path.exists(issue_filepath):
         raise ValueError("A issue YAML file already exists at {issue_filepath}.")
+
     # Create issue dir if not existing
     issue_dir = os.path.dirname(issue_filepath)
     os.makedirs(issue_dir, exist_ok=True)
+
     # Write issue file
     write_issue(filepath=issue_filepath)
     print(f"An empty issue YAML file for station {station_name} has been created .")

disdrodb/l0/check_configs.py

@@ -22,7 +22,7 @@ import os
 from typing import Optional, Union
 
 import numpy as np
-from pydantic import BaseModel, ValidationError, field_validator, model_validator
+from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator
 
 from disdrodb.api.configs import available_sensor_names, get_sensor_configs_dir, read_config_file
 from disdrodb.l0.standards import (
@@ -47,7 +47,7 @@ CONFIG_FILES_LIST = [
 ]
 
 
-def _check_yaml_files_exists(sensor_name: str) -> None:
+def check_yaml_files_exists(sensor_name: str) -> None:
     """Check if all L0 config YAML files exist.
 
     Parameters
@@ -64,7 +64,7 @@ def _check_yaml_files_exists(sensor_name: str) -> None:
         raise FileNotFoundError(f"Missing YAML files {missing_keys_text} in {config_dir} for sensor {sensor_name}.")
 
 
-def _check_variable_consistency(sensor_name: str) -> None:
+def check_variable_consistency(sensor_name: str) -> None:
     """
     Check variable consistency across config files.
 
@@ -126,7 +126,7 @@ def _schema_error(object_to_validate: Union[str, list], schema: BaseModel, messa
 
 
 class L0BEncodingSchema(BaseModel):
-    """Pydantic model for DISDRODB L0B encodings."""
+    """Pydantic model for DISDRODB netCDF encodings."""
 
     contiguous: bool
     dtype: str
@@ -134,7 +134,7 @@ class L0BEncodingSchema(BaseModel):
     complevel: int
     shuffle: bool
     fletcher32: bool
-    _FillValue: Optional[Union[int, float]]
+    FillValue: Optional[Union[int, float]] = Field(default=None, alias="_FillValue")
     chunksizes: Optional[Union[int, list[int]]]
 
     # if contiguous=False, chunksizes specified, otherwise should be not !
@@ -167,6 +167,39 @@ class L0BEncodingSchema(BaseModel):
             raise ValueError("'fletcher32' must be set to False if 'contiguous' is True")
         return values
 
+    # if dtype is integer/unsigned integer, _FillValue must be specified and valid
+    @model_validator(mode="before")
+    def check_integer_fillvalue(cls, values):
+        """Check that integer dtypes have valid _FillValue."""
+        dtype = values.get("dtype")
+        fill_value = values.get("_FillValue", None)
+        integer_types = ["int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64"]
+        # Check if dtype is an integer type
+        if dtype in integer_types:
+            # _FillValue must be specified for integer types
+            if fill_value is None:
+                raise ValueError(f"'_FillValue' must be specified for integer dtype '{dtype}'")
+
+            # Check that _FillValue is within valid range for the dtype
+            dtype_info = np.iinfo(dtype)
+            max_value = dtype_info.max
+            # min_value = dtype_info.min
+
+            # if not (min_value <= fill_value <= max_value):
+            #     raise ValueError(
+            #         f"'_FillValue' ({fill_value}) is out of range for dtype '{dtype}'. "
+            #         f"Valid range is [{min_value}, {max_value}]",
+            #     )
+
+            # Check that _FillValue corresponds to the maximum allowed value
+            if fill_value != max_value:
+                raise ValueError(
+                    f"'_FillValue' ({fill_value}) should be set to the maximum allowed value "
+                    f"({max_value}) for integer dtype '{dtype}'",
+                )
+
+        return values
+
 
 def check_l0b_encoding(sensor_name: str) -> None:
     """Check ``l0b_encodings.yml`` file based on the schema defined in the class ``L0BEncodingSchema``.
@@ -176,7 +209,7 @@ def check_l0b_encoding(sensor_name: str) -> None:
     sensor_name : str
         Name of the sensor.
     """
-    data = read_config_file(sensor_name, product="L0A", filename="l0b_encodings.yml")
+    data = read_config_file(sensor_name, product="L0B", filename="l0b_encodings.yml")
 
     # check that the second level of the dictionary match the schema
     for key, value in data.items():
@@ -234,7 +267,7 @@ class RawDataFormatSchema(BaseModel):
         return None
 
 
-def _check_raw_data_format(sensor_name: str) -> None:
+def check_raw_data_format(sensor_name: str) -> None:
     """Check ``raw_data_format.yml`` file based on the schema defined in the class ``RawDataFormatSchema``.
 
     Parameters
@@ -253,7 +286,7 @@ def _check_raw_data_format(sensor_name: str) -> None:
         )
 
 
-def _check_cf_attributes(sensor_name: str) -> None:
+def check_cf_attributes(sensor_name: str) -> None:
     """Check that the ``l0b_cf_attrs.yml`` description, long_name and units values are strings.
 
     Parameters
@@ -268,7 +301,7 @@ def _check_cf_attributes(sensor_name: str) -> None:
                 raise ValueError(f"Wrong value for {key} in {var} for sensor {sensor_name}.")
 
 
-def _check_bin_consistency(sensor_name: str) -> None:
+def check_bin_consistency(sensor_name: str) -> None:
     """Check bin consistency from config file.
 
     Do not check the first and last bin !
@@ -313,7 +346,7 @@ def _check_bin_consistency(sensor_name: str) -> None:
         )
 
 
-def _check_raw_array(sensor_name: str) -> None:
+def check_raw_array(sensor_name: str) -> None:
     """Check raw array consistency from config file.
 
     Parameters
@@ -363,14 +396,14 @@ def check_sensor_configs(sensor_name: str) -> None:
     sensor_name : str
         Name of the sensor.
     """
-    _check_yaml_files_exists(sensor_name)
-    _check_variable_consistency(sensor_name)
+    check_yaml_files_exists(sensor_name)
+    check_variable_consistency(sensor_name)
     check_l0b_encoding(sensor_name=sensor_name)
     check_l0a_encoding(sensor_name=sensor_name)
-    _check_raw_data_format(sensor_name=sensor_name)
-    _check_cf_attributes(sensor_name=sensor_name)
-    _check_bin_consistency(sensor_name=sensor_name)
-    _check_raw_array(sensor_name=sensor_name)
+    check_raw_data_format(sensor_name=sensor_name)
+    check_cf_attributes(sensor_name=sensor_name)
+    check_bin_consistency(sensor_name=sensor_name)
+    check_raw_array(sensor_name=sensor_name)
 
 
 def check_all_sensors_configs() -> None:

disdrodb/l0/configs/LPM/l0a_encodings.yml

@@ -15,7 +15,7 @@ reflectivity: "float32"
 quality_index: "float32" # 'uint8'
 max_hail_diameter: "float32"
 laser_status: "float32" # 'uint8'
-static_signal: "float32" # 'uint8'
+static_signal_status: "float32" # 'uint8'
 laser_temperature_analog_status: "float32" # 'uint8'
 laser_temperature_digital_status: "float32" # 'uint8'
 laser_current_analog_status: "float32" # 'uint8'
@@ -29,7 +29,7 @@ current_heating_house_status: "float32" # 'uint8'
 current_heating_heads_status: "float32" # 'uint8'
 current_heating_carriers_status: "float32" # 'uint8'
 control_output_laser_power_status: "float32" # 'uint8'
-reserve_status: "float32" # 'uint8'
+reserved_status: "float32" # 'uint8'
 temperature_interior: "float32" # 'uint16'
 laser_temperature: "float32" # 'uint16'
 laser_current_average: "float32" # 'uint16'

disdrodb/l0/configs/LPM/l0b_cf_attrs.yml

@@ -66,7 +66,7 @@ laser_status:
   description: Status Laser (OK/on:0, off:1)
   long_name: Status laser
   units: ""
-static_signal:
+static_signal_status:
   description: Static signal (OK:0, Error:1)
   long_name: Static signal
   units: ""
@@ -122,7 +122,7 @@ control_output_laser_power_status:
   description: Status Control output laser power (OK:0, warning:1)
   long_name: Status control output laser
   units: ""
-reserve_status:
+reserved_status:
   description: Reserve Status (0)
   long_name: Reserve status
   units: ""

disdrodb/l0/configs/LPM/l0b_encodings.yml

@@ -184,7 +184,7 @@ laser_status:
   contiguous: false
   chunksizes: 5000
   _FillValue: 255
-static_signal:
+static_signal_status:
   dtype: uint8
   zlib: true
   complevel: 3
@@ -310,7 +310,7 @@ control_output_laser_power_status:
   contiguous: false
   chunksizes: 5000
   _FillValue: 255
-reserve_status:
+reserved_status:
   dtype: uint8
   zlib: true
   complevel: 3

disdrodb/l0/configs/LPM/raw_data_format.yml

@@ -213,7 +213,7 @@ laser_status:
     - 0
     - 1
   field_number: "22"
-static_signal:
+static_signal_status:
   n_digits: 1
   n_characters: 1
   n_decimals: 1
@@ -389,7 +389,7 @@ control_output_laser_power_status:
     - 0
     - 1
   field_number: "36"
-reserve_status:
+reserved_status:
   n_digits: 1
   n_characters: 1
   n_decimals: 1

disdrodb/l0/configs/PWS100/l0b_encodings.yml

@@ -68,6 +68,7 @@ relative_humidity:
   shuffle: true
   fletcher32: false
   contiguous: false
+  _FillValue: 65535
   chunksizes: 5000
 wetbulb_temperature:
   dtype: uint16

disdrodb/l0/configs/SWS250/bins_diameter.yml

@@ -0,0 +1,108 @@
+center:
+  0: 0.2
+  1: 0.45
+  2: 0.55
+  3: 0.65
+  4: 0.75
+  5: 0.85
+  6: 0.95
+  7: 1.05
+  8: 1.15
+  9: 1.25
+  10: 1.35
+  11: 1.5
+  12: 1.75
+  13: 2.08
+  14: 2.475
+  15: 2.945
+  16: 3.5
+  17: 4.16
+  18: 4.95
+  19: 5.89
+  20: 6.7
+bounds:
+  0:
+    - 0.0
+    - 0.4
+  1:
+    - 0.4
+    - 0.5
+  2:
+    - 0.5
+    - 0.6
+  3:
+    - 0.6
+    - 0.7
+  4:
+    - 0.7
+    - 0.8
+  5:
+    - 0.8
+    - 0.9
+  6:
+    - 0.9
+    - 1.0
+  7:
+    - 1.0
+    - 1.1
+  8:
+    - 1.1
+    - 1.2
+  9:
+    - 1.2
+    - 1.3
+  10:
+    - 1.3
+    - 1.4
+  11:
+    - 1.4
+    - 1.6
+  12:
+    - 1.6
+    - 1.9
+  13:
+    - 1.9
+    - 2.26
+  14:
+    - 2.26
+    - 2.69
+  15:
+    - 2.69
+    - 3.2
+  16:
+    - 3.2
+    - 3.8
+  17:
+    - 3.8
+    - 4.52
+  18:
+    - 4.52
+    - 5.38
+  19:
+    - 5.38
+    - 6.4
+  20:
+    - 6.4
+    - 7.0
+width:
+  0: 0.4
+  1: 0.1
+  2: 0.1
+  3: 0.1
+  4: 0.1
+  5: 0.1
+  6: 0.1
+  7: 0.1
+  8: 0.1
+  9: 0.1
+  10: 0.1
+  11: 0.2
+  12: 0.3
+  13: 0.36
+  14: 0.43
+  15: 0.51
+  16: 0.6
+  17: 0.72
+  18: 0.86
+  19: 1.02
+  20: 0.6