disdrodb 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

disdrodb/__init__.py

@@ -73,7 +73,7 @@ DIAMETER_COORDS = ["diameter_bin_center", "diameter_bin_width", "diameter_bin_lo
 VELOCITY_COORDS = ["velocity_bin_center", "velocity_bin_width", "velocity_bin_lower", "velocity_bin_upper"]
 VELOCITY_DIMENSION = "velocity_bin_center"
 DIAMETER_DIMENSION = "diameter_bin_center"
-OPTICAL_SENSORS = ["PARSIVEL", "PARSIVEL2", "LPM"]
+OPTICAL_SENSORS = ["PARSIVEL", "PARSIVEL2", "LPM", "PWS100"]
 IMPACT_SENSORS = ["RD80"]
 
 

disdrodb/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.0'
-__version_tuple__ = version_tuple = (0, 1, 0)
+__version__ = version = '0.1.2'
+__version_tuple__ = version_tuple = (0, 1, 2)

disdrodb/api/io.py

@@ -141,6 +141,7 @@ def open_dataset(
     product_kwargs=None,
     debugging_mode: bool = False,
     data_archive_dir: Optional[str] = None,
+    parallel=False,
     **open_kwargs,
 ):
     """Retrieve DISDRODB product files for a give station.
@@ -205,8 +206,17 @@ def open_dataset(
     # Open DISDRODB netCDF files using xarray
     # - TODO: parallel option and add closers !
     # - decode_timedelta -- > sample_interval not decoded to timedelta !
-    list_ds = [xr.open_dataset(fpath, decode_timedelta=False, **open_kwargs) for fpath in filepaths]
-    ds = xr.concat(list_ds, dim="time")
+    # list_ds = [xr.open_dataset(fpath, decode_timedelta=False, **open_kwargs) for fpath in filepaths]
+    # ds = xr.concat(list_ds, dim="time")
+    ds = xr.open_mfdataset(
+        filepaths,
+        engine="netcdf4",
+        combine="nested",  # 'by_coords',
+        concat_dim="time",
+        decode_timedelta=False,
+        parallel=parallel,
+        **open_kwargs,
+    )
     return ds
 
 

disdrodb/data_transfer/download_data.py

@@ -21,6 +21,8 @@
 import logging
 import os
 import shutil
+import subprocess
+import urllib.parse
 from typing import Optional, Union
 
 import click
@@ -213,7 +215,7 @@ def download_station(
         check_exists=True,
     )
     # Download data
-    _download_station_data(metadata_filepath, data_archive_dir=data_archive_dir, force=force)
+    download_station_data(metadata_filepath, data_archive_dir=data_archive_dir, force=force)
 
 
 def _is_valid_disdrodb_data_url(disdrodb_data_url):
@@ -228,13 +230,25 @@ def _extract_station_files(zip_filepath, station_dir):
         os.remove(zip_filepath)
 
 
-def _download_station_data(metadata_filepath: str, data_archive_dir: str, force: bool = False) -> None:
+def check_consistent_station_name(metadata_filepath, station_name):
+    """Check consistent station_name between YAML file name and metadata key."""
+    # Check consistent station name
+    expected_station_name = os.path.basename(metadata_filepath).replace(".yml", "")
+    if station_name and str(station_name) != str(expected_station_name):
+        raise ValueError(f"Inconsistent station_name values in the {metadata_filepath} file. Download aborted.")
+    return station_name
+
+
+def download_station_data(metadata_filepath: str, data_archive_dir: str, force: bool = False) -> None:
     """Download and unzip the station data .
 
     Parameters
     ----------
     metadata_filepaths : str
         Metadata file path.
+    data_archive_dir : str (optional)
+        DISDRODB Data Archive directory. Format: ``<...>/DISDRODB``.
+        If ``None`` (the default), the disdrodb config variable ``data_archive_dir`` is used.
     force : bool, optional
         If ``True``, delete existing files and redownload it. The default value is ``False``.
 
@@ -247,7 +261,7 @@ def _download_station_data(metadata_filepath: str, data_archive_dir: str, force:
     campaign_name = metadata_dict["campaign_name"]
     station_name = metadata_dict["station_name"]
     station_name = check_consistent_station_name(metadata_filepath, station_name)
-    # Define the destination local filepath path
+    # Define the path to the station RAW data directory
     station_dir = define_station_dir(
         data_archive_dir=data_archive_dir,
         data_source=data_source,
@@ -259,19 +273,136 @@ def _download_station_data(metadata_filepath: str, data_archive_dir: str, force:
     disdrodb_data_url = metadata_dict.get("disdrodb_data_url", None)
     if not _is_valid_disdrodb_data_url(disdrodb_data_url):
         raise ValueError(f"Invalid disdrodb_data_url '{disdrodb_data_url}' for station {station_name}")
-    # Download file
-    zip_filepath = _download_file_from_url(disdrodb_data_url, dst_dir=station_dir, force=force)
-    # Extract the stations files from the downloaded station.zip file
-    _extract_station_files(zip_filepath, station_dir=station_dir)
 
+    # Download files
+    # - Option 1: Zip file from Zenodo containing all station raw data
+    if disdrodb_data_url.startswith("https://zenodo.org/"):
+        download_zenodo_zip_file(url=disdrodb_data_url, dst_dir=station_dir, force=force)
+    # - Option 2: Recursive download from a web server via HTTP or HTTPS.
+    elif disdrodb_data_url.startswith("http"):
+        download_web_server_data(url=disdrodb_data_url, dst_dir=station_dir, force=force, verbose=True)
+    else:
+        raise NotImplementedError(f"Open a GitHub Issue to enable the download of data from {disdrodb_data_url}.")
 
-def check_consistent_station_name(metadata_filepath, station_name):
-    """Check consistent station_name between YAML file name and metadata key."""
-    # Check consistent station name
-    expected_station_name = os.path.basename(metadata_filepath).replace(".yml", "")
-    if station_name and str(station_name) != str(expected_station_name):
-        raise ValueError(f"Inconsistent station_name values in the {metadata_filepath} file. Download aborted.")
-    return station_name
+
+####-----------------------------------------------------------------------------------------.
+#### Download from Web Server via HTTP or HTTPS
+
+
+def download_web_server_data(url: str, dst_dir: str, force=True, verbose=True) -> None:
+    """Download data from a web server via HTTP or HTTPS.
+
+    Use the system's wget command to recursively download all files and subdirectories
+    under the given HTTPS “directory” URL. Works on both Windows and Linux, provided
+    that wget is installed and on the PATH.
+
+    1. Ensure wget is available.
+    2. Normalize URL to end with '/'.
+    3. Compute cut-dirs so that only the last segment of the path remains locally.
+    4. Build and run the wget command.
+
+    Example:
+        download_with_wget("https://ruisdael.citg.tudelft.nl/parsivel/PAR001_Cabauw/2021/202101/")
+        # → Creates a local folder "202101/" with all files and subfolders.
+    """
+    # 1. Ensure wget exists
+    ensure_wget_available()
+
+    # 2. Normalize URL
+    url = ensure_trailing_slash(url)
+
+    # 3. Compute cut-dirs so that only the last URL segment remains locally
+    cut_dirs = compute_cut_dirs(url)
+
+    # 4. Create destination directory if needed
+    os.makedirs(dst_dir, exist_ok=True)
+
+    # 5. Build wget command
+    cmd = build_webserver_wget_command(url, cut_dirs=cut_dirs, dst_dir=dst_dir, force=force, verbose=verbose)
+
+    # 6. Run wget command
+    try:
+        subprocess.run(cmd, check=True)
+    except subprocess.CalledProcessError as e:
+        raise subprocess.CalledProcessError(
+            returncode=e.returncode,
+            cmd=e.cmd,
+            output=e.output,
+            stderr=e.stderr,
+        )
+
+
+def ensure_wget_available() -> None:
+    """Raise FileNotFoundError if 'wget' is not on the system PATH."""
+    if shutil.which("wget") is None:
+        raise FileNotFoundError("The WGET software was not found. Please install WGET or add it to PATH.")
+
+
+def ensure_trailing_slash(url: str) -> str:
+    """Return `url` guaranteed to end with a slash."""
+    return url if url.endswith("/") else url.rstrip("/") + "/"
+
+
+def compute_cut_dirs(url: str) -> int:
+    """Compute the wget cut_dirs value to download directly in `dst_dir`.
+
+    Given a URL ending with '/', compute the total number of path segments.
+    By returning len(segments), we strip away all of them—so that files
+    within that final directory land directly in `dst_dir` without creating
+    an extra subfolder.
+    """
+    parsed = urllib.parse.urlparse(url)
+    path = parsed.path.strip("/")  # remove leading/trailing '/'
+    segments = path.split("/") if path else []
+    return len(segments)
+
+
+def build_webserver_wget_command(url: str, cut_dirs: int, dst_dir: str, force: bool, verbose: bool) -> list[str]:
+    """Construct the wget command list for subprocess.run.
+
+    Notes
+    -----
+    The following wget arguments are used
+      - -q         : quiet mode (no detailed progress)
+      - -r         : recursive
+      - -np        : no parent
+      - -nH        : no host directories
+      - --timestamping: download missing files or when remote version is newer
+      - --cut-dirs : strip all but the last path segment from the remote path
+      - -P dst_dir : download into `dst_dir`
+      - url
+    """
+    cmd = ["wget"]
+    if verbose:
+        cmd.append("-q")
+    cmd += [
+        "-r",
+        "-np",
+        "-nH",
+        f"--cut-dirs={cut_dirs}",
+    ]
+    if force:
+        cmd.append("--timestamping")  # -N
+
+    # Define source and destination directory
+    cmd += [
+        "-P",
+        dst_dir,
+        url,
+    ]
+    return cmd
+
+
+####--------------------------------------------------------------------.
+#### Download from Zenodo
+
+
+def download_zenodo_zip_file(url, dst_dir, force):
+    """Download zip file from zenodo and extract station raw data."""
+    # Download zip file
+    zip_filepath = _download_file_from_url(url, dst_dir=dst_dir, force=force)
+    # Extract the stations files from the downloaded station.zip file
+    _extract_station_files(zip_filepath, station_dir=dst_dir)
 
 
 def _download_file_from_url(url: str, dst_dir: str, force: bool = False) -> str:

disdrodb/l0/check_standards.py

@@ -80,7 +80,12 @@ def _check_valid_values(df, dict_valid_values):
         raise ValueError(f"Columns {list_wrong_columns} have invalid values.")
 
 
-def _check_raw_fields_available(df: pd.DataFrame, sensor_name: str, logger=None, verbose: bool = False) -> None:
+def _check_raw_fields_available(
+    df: pd.DataFrame,
+    sensor_name: str,  # noqa: ARG001
+    logger=None,  # noqa: ARG001
+    verbose: bool = False,  # noqa: ARG001
+) -> None:
     """Check the presence of the raw spectrum data according to the type of sensor.
 
     Parameters
@@ -95,22 +100,22 @@ def _check_raw_fields_available(df: pd.DataFrame, sensor_name: str, logger=None,
     ValueError
         Error if the ``raw_drop_number`` field is missing.
     """
-    from disdrodb.l0.standards import get_raw_array_nvalues
-
-    # Retrieve raw arrays that could be available (based on sensor_name)
-    n_bins_dict = get_raw_array_nvalues(sensor_name=sensor_name)
-    raw_vars = np.array(list(n_bins_dict.keys()))
+    # from disdrodb.l0.standards import get_raw_array_nvalues
 
     # Check that raw_drop_number is present
     if "raw_drop_number" not in df.columns:
         msg = "The 'raw_drop_number' column is not present in the dataframe."
         raise ValueError(msg)
 
+    # Retrieve raw arrays that could be available (based on sensor_name)
+    #  n_bins_dict = get_raw_array_nvalues(sensor_name=sensor_name)
+
     # Report additional raw arrays that are missing
-    missing_vars = raw_vars[np.isin(raw_vars, list(df.columns), invert=True)]
-    if len(missing_vars) > 0:
-        msg = f"The following raw array variable are missing: {missing_vars}"
-        log_info(logger=logger, msg=msg, verbose=verbose)
+    # raw_vars = np.array(list(n_bins_dict.keys()))
+    # missing_vars = raw_vars[np.isin(raw_vars, list(df.columns), invert=True)]
+    # if len(missing_vars) > 0:
+    #     msg = f"The following raw array variable are missing: {missing_vars}"
+    #     log_info(logger=logger, msg=msg, verbose=verbose)
 
 
 def check_l0a_column_names(df: pd.DataFrame, sensor_name: str) -> None:

disdrodb/l0/configs/LPM/bins_diameter.yml

@@ -20,7 +20,7 @@ center:
   18: 6.75
   19: 7.25
   20: 7.75
-  21: 54
+  21: 9
 bounds:
   0:
     - 0.125
@@ -87,7 +87,7 @@ bounds:
     - 8.0
   21:
     - 8.0
-    - 100
+    - 10.0
 width:
   0: 0.125
   1: 0.125
@@ -110,4 +110,4 @@ width:
   18: 0.5
   19: 0.5
   20: 0.5
-  21: 92
+  21: 2

disdrodb/l0/configs/LPM/l0a_encodings.yml

@@ -74,7 +74,7 @@ number_particles_class_8_internal_data: "float32"
 number_particles_class_9: "float32" # 'uint16'
 number_particles_class_9_internal_data: "float32"
 raw_drop_number: "str"
-# '521': air_temperature
-# '522': relative_humidity
-# '523': wind_speed
-# '524': wind_direction
+air_temperature: "float32"
+relative_humidity: "float32"
+wind_speed: "float32"
+wind_direction: "float32"

disdrodb/l0/configs/LPM/l0b_cf_attrs.yml

@@ -127,13 +127,13 @@ reserve_status:
   long_name: Reserve status
   units: ""
 temperature_interior:
-  description: "Interior temperature [\xB0C] (NNN)"
+  description: "Interior temperature [C] (NNN)"
   long_name: Interior temperature
-  units: "\xB0C"
+  units: "C"
 laser_temperature:
-  description: "Temperature of laser driver 0-80\xB0C (NN)"
+  description: "Temperature of laser driver 0-80C (NN)"
   long_name: Temperature of laser driver
-  units: "\xB0C"
+  units: "C"
 laser_current_average:
   description: Mean value laser current [1/100 mA] (NNNN)
   long_name: Mean value laser current
@@ -159,9 +159,9 @@ current_heating_pane_receiver_head:
   long_name: Current pane heating receiver head
   units: mA
 temperature_ambient:
-  description: "Ambient temperature [\xB0C] (NNN.N)"
+  description: "Ambient temperature [C] (NNN.N)"
   long_name: Ambient temperature
-  units: "\xB0C"
+  units: "C"
 current_heating_voltage_supply:
   description:
     Voltage Heating supply [1/10 V] (only 5.4110.x1.xxx, otherwise "999")
@@ -308,3 +308,19 @@ raw_drop_number:
   description: Precipitation spectrum
   long_name: Precipitation spectrum
   units: ""
+air_temperature:
+  description: "Air temperature in degrees Celsius (C)"
+  long_name: Air temperature
+  units: "C"
+relative_humidity:
+  description: "Relative humidity in percent (%)"
+  long_name: Relative humidity
+  units: "%"
+wind_speed:
+  description: "Wind speed in m/s"
+  long_name: Wind speed
+  units: "m/s"
+wind_direction:
+  description: "Wind direction in degrees (0-360)"
+  long_name: Wind direction
+  units: "degrees"

disdrodb/l0/configs/LPM/l0b_encodings.yml

@@ -693,3 +693,44 @@ raw_drop_number:
     - 5000
     - 22 # diameter
     - 20 # velocity
+air_temperature:
+  dtype: uint16
+  scale_factor: 0.1
+  add_offset: -99.9
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 5000
+relative_humidity:
+  dtype: uint16
+  scale_factor: 0.01
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 5000
+wind_speed:
+  dtype: uint16
+  scale_factor: 0.1
+  add_offset: -99.9
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 5000
+wind_direction:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 5000

disdrodb/l0/configs/LPM/raw_data_format.yml

@@ -828,3 +828,43 @@ raw_drop_number:
     - velocity_bin_center
   n_values: 440
   field_number: "81"
+air_temperature:
+  n_digits: 4
+  n_characters: 5
+  n_decimals: 1
+  n_naturals: 2
+  data_range:
+    - -40
+    - 70
+  nan_flags: 99999
+  field_number: "521"
+relative_humidity:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 0
+  n_naturals: 5
+  data_range:
+    - 0
+    - 99999
+  nan_flags: 99999
+  field_number: "522"
+wind_speed:
+  n_digits: 3
+  n_characters: 4
+  n_decimals: 1
+  n_naturals: 2
+  data_range:
+    - 0
+    - 60
+  nan_flags: null
+  field_number: "523"
+wind_direction:
+  n_digits: 3
+  n_characters: 3
+  n_decimals: 0
+  n_naturals: 3
+  data_range:
+    - 0
+    - 360
+  nan_flags: 999
+  field_number: "524"

disdrodb/l0/configs/PARSIVEL/l0b_cf_attrs.yml

@@ -47,7 +47,7 @@ number_particles:
 sensor_temperature:
   description: Temperature in sensor housing
   long_name: Temperature of the sensor
-  units: "\xB0C"
+  units: "C"
 sensor_serial_number:
   description: Sensor serial number
   long_name: Serial number of the sensor

disdrodb/l0/configs/PARSIVEL/raw_data_format.yml

@@ -71,7 +71,7 @@ mor_visibility:
   n_naturals: 4
   data_range:
     - 0
-    - 20000
+    - 9999
   nan_flags: null
   field_number: 08
 sample_interval:

disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml

@@ -37,3 +37,7 @@ list_particles: "str"
 raw_drop_concentration: "str"
 raw_drop_average_velocity: "str"
 raw_drop_number: "str"
+air_temperature: "float32"
+relative_humidity: "float32"
+wind_speed: "float32"
+wind_direction: "float32"

disdrodb/l0/configs/PARSIVEL2/l0b_cf_attrs.yml

@@ -47,7 +47,7 @@ number_particles:
 sensor_temperature:
   description: Temperature in sensor housing
   long_name: Temperature of the sensor
-  units: "\xB0C"
+  units: "C"
 sensor_serial_number:
   description: Sensor serial number
   long_name: Serial number of the sensor
@@ -105,15 +105,15 @@ error_code:
 sensor_temperature_pcb:
   description: Temperature in printed circuit board
   long_name: Sensor PCB temperature
-  units: "\xB0C"
+  units: "C"
 sensor_temperature_receiver:
   description: Temperature in right sensor head
   long_name: Sensor receiver temperature
-  units: "\xB0C"
+  units: "C"
 sensor_temperature_trasmitter:
   description: Temperature in left sensor head
   long_name: Sensor trasmitter temperature
-  units: "\xB0C"
+  units: "C"
 rainfall_rate_16_bit_30:
   description: Rainfall rate
   long_name: Rainfall rate max 30 mm/h 16 bit
@@ -158,3 +158,19 @@ raw_drop_number:
   description: Drop counts per diameter and velocity class
   long_name: Raw drop number
   units: ""
+air_temperature:
+  description: "Air temperature in degrees Celsius (C)"
+  long_name: Air temperature
+  units: "C"
+relative_humidity:
+  description: "Relative humidity in percent (%)"
+  long_name: Relative humidity
+  units: "%"
+wind_speed:
+  description: "Wind speed in m/s"
+  long_name: Wind speed
+  units: "m/s"
+wind_direction:
+  description: "Wind direction in degrees (0-360)"
+  long_name: Wind direction
+  units: "degrees"

disdrodb/l0/configs/PARSIVEL2/l0b_encodings.yml

@@ -331,3 +331,44 @@ raw_drop_number:
     - 5000
     - 32
     - 32
+air_temperature:
+  dtype: uint16
+  scale_factor: 0.1
+  add_offset: -99.9
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 5000
+relative_humidity:
+  dtype: uint16
+  scale_factor: 0.01
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 5000
+wind_speed:
+  dtype: uint16
+  scale_factor: 0.1
+  add_offset: -99.9
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 5000
+wind_direction:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 5000

disdrodb/l0/configs/PARSIVEL2/raw_data_format.yml

@@ -65,10 +65,10 @@ reflectivity_32bit:
   nan_flags: null
   field_number: "07"
 mor_visibility:
-  n_digits: 4
-  n_characters: 4
+  n_digits: 5
+  n_characters: 5
   n_decimals: 0
-  n_naturals: 4
+  n_naturals: 5
   data_range:
     - 0
     - 20000
@@ -270,8 +270,8 @@ sensor_temperature_trasmitter:
 rainfall_rate_16_bit_30:
   n_digits: 5
   n_characters: 6
-  n_decimals: 2
-  n_naturals: 3
+  n_decimals: 3
+  n_naturals: 2
   data_range:
     - 0
     - 30
@@ -280,8 +280,8 @@ rainfall_rate_16_bit_30:
 rainfall_rate_16_bit_1200:
   n_digits: 5
   n_characters: 6
-  n_decimals: 4
-  n_naturals: 1
+  n_decimals: 1
+  n_naturals: 4
   data_range:
     - 0
     - 1200
@@ -310,11 +310,11 @@ reflectivity_16bit:
 rain_kinetic_energy:
   n_digits: 6
   n_characters: 7
-  n_decimals: 3
-  n_naturals: 3
+  n_decimals: 2
+  n_naturals: 4
   data_range:
     - 0
-    - 999.999
+    - 9999.99
   nan_flags: null
   field_number: "34"
 snowfall_rate:
@@ -379,3 +379,43 @@ raw_drop_number:
     - diameter_bin_center
   n_values: 1024
   field_number: "93"
+air_temperature:
+  n_digits: 4
+  n_characters: 5
+  n_decimals: 1
+  n_naturals: 2
+  data_range:
+    - -40
+    - 70
+  nan_flags: 99999
+  field_number: "521"
+relative_humidity:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 0
+  n_naturals: 5
+  data_range:
+    - 0
+    - 99999
+  nan_flags: 99999
+  field_number: "522"
+wind_speed:
+  n_digits: 3
+  n_characters: 4
+  n_decimals: 1
+  n_naturals: 2
+  data_range:
+    - 0
+    - 60
+  nan_flags: null
+  field_number: "523"
+wind_direction:
+  n_digits: 3
+  n_characters: 3
+  n_decimals: 0
+  n_naturals: 3
+  data_range:
+    - 0
+    - 360
+  nan_flags: 999
+  field_number: "524"