disdrodb 0.1.1__tar.gz → 0.1.2__tar.gz

{disdrodb-0.1.1/disdrodb.egg-info → disdrodb-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: disdrodb
-Version: 0.1.1
+Version: 0.1.2
 Summary: disdrodb provides tools to download, standardize, share and analyze global disdrometer data.
 Author: Gionata Ghiggi
 Project-URL: homepage, https://github.com/ltelab/disdrodb

{disdrodb-0.1.1 → disdrodb-0.1.2}/disdrodb/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.1'
-__version_tuple__ = version_tuple = (0, 1, 1)
+__version__ = version = '0.1.2'
+__version_tuple__ = version_tuple = (0, 1, 2)

{disdrodb-0.1.1 → disdrodb-0.1.2}/disdrodb/data_transfer/download_data.py

@@ -21,6 +21,8 @@
 import logging
 import os
 import shutil
+import subprocess
+import urllib.parse
 from typing import Optional, Union
 
 import click
@@ -213,7 +215,7 @@ def download_station(
         check_exists=True,
     )
     # Download data
-    _download_station_data(metadata_filepath, data_archive_dir=data_archive_dir, force=force)
+    download_station_data(metadata_filepath, data_archive_dir=data_archive_dir, force=force)
 
 
 def _is_valid_disdrodb_data_url(disdrodb_data_url):
@@ -228,13 +230,25 @@ def _extract_station_files(zip_filepath, station_dir):
         os.remove(zip_filepath)
 
 
-def _download_station_data(metadata_filepath: str, data_archive_dir: str, force: bool = False) -> None:
+def check_consistent_station_name(metadata_filepath, station_name):
+    """Check consistent station_name between YAML file name and metadata key."""
+    # Check consistent station name
+    expected_station_name = os.path.basename(metadata_filepath).replace(".yml", "")
+    if station_name and str(station_name) != str(expected_station_name):
+        raise ValueError(f"Inconsistent station_name values in the {metadata_filepath} file. Download aborted.")
+    return station_name
+
+
+def download_station_data(metadata_filepath: str, data_archive_dir: str, force: bool = False) -> None:
     """Download and unzip the station data .
 
     Parameters
     ----------
     metadata_filepaths : str
         Metadata file path.
+    data_archive_dir : str (optional)
+        DISDRODB Data Archive directory. Format: ``<...>/DISDRODB``.
+        If ``None`` (the default), the disdrodb config variable ``data_archive_dir`` is used.
     force : bool, optional
         If ``True``, delete existing files and redownload it. The default value is ``False``.
 
@@ -247,7 +261,7 @@ def _download_station_data(metadata_filepath: str, data_archive_dir: str, force:
     campaign_name = metadata_dict["campaign_name"]
     station_name = metadata_dict["station_name"]
     station_name = check_consistent_station_name(metadata_filepath, station_name)
-    # Define the destination local filepath path
+    # Define the path to the station RAW data directory
     station_dir = define_station_dir(
         data_archive_dir=data_archive_dir,
         data_source=data_source,
@@ -259,19 +273,136 @@ def _download_station_data(metadata_filepath: str, data_archive_dir: str, force:
     disdrodb_data_url = metadata_dict.get("disdrodb_data_url", None)
     if not _is_valid_disdrodb_data_url(disdrodb_data_url):
         raise ValueError(f"Invalid disdrodb_data_url '{disdrodb_data_url}' for station {station_name}")
-    # Download file
-    zip_filepath = _download_file_from_url(disdrodb_data_url, dst_dir=station_dir, force=force)
-    # Extract the stations files from the downloaded station.zip file
-    _extract_station_files(zip_filepath, station_dir=station_dir)
 
+    # Download files
+    # - Option 1: Zip file from Zenodo containing all station raw data
+    if disdrodb_data_url.startswith("https://zenodo.org/"):
+        download_zenodo_zip_file(url=disdrodb_data_url, dst_dir=station_dir, force=force)
+    # - Option 2: Recursive download from a web server via HTTP or HTTPS.
+    elif disdrodb_data_url.startswith("http"):
+        download_web_server_data(url=disdrodb_data_url, dst_dir=station_dir, force=force, verbose=True)
+    else:
+        raise NotImplementedError(f"Open a GitHub Issue to enable the download of data from {disdrodb_data_url}.")
 
-def check_consistent_station_name(metadata_filepath, station_name):
-    """Check consistent station_name between YAML file name and metadata key."""
-    # Check consistent station name
-    expected_station_name = os.path.basename(metadata_filepath).replace(".yml", "")
-    if station_name and str(station_name) != str(expected_station_name):
-        raise ValueError(f"Inconsistent station_name values in the {metadata_filepath} file. Download aborted.")
-    return station_name
+
+####-----------------------------------------------------------------------------------------.
+#### Download from Web Server via HTTP or HTTPS
+
+
+def download_web_server_data(url: str, dst_dir: str, force=True, verbose=True) -> None:
+    """Download data from a web server via HTTP or HTTPS.
+
+    Use the system's wget command to recursively download all files and subdirectories
+    under the given HTTPS “directory” URL. Works on both Windows and Linux, provided
+    that wget is installed and on the PATH.
+
+    1. Ensure wget is available.
+    2. Normalize URL to end with '/'.
+    3. Compute cut-dirs so that only the last segment of the path remains locally.
+    4. Build and run the wget command.
+
+    Example:
+        download_with_wget("https://ruisdael.citg.tudelft.nl/parsivel/PAR001_Cabauw/2021/202101/")
+        # → Creates a local folder "202101/" with all files and subfolders.
+    """
+    # 1. Ensure wget exists
+    ensure_wget_available()
+
+    # 2. Normalize URL
+    url = ensure_trailing_slash(url)
+
+    # 3. Compute cut-dirs so that only the last URL segment remains locally
+    cut_dirs = compute_cut_dirs(url)
+
+    # 4. Create destination directory if needed
+    os.makedirs(dst_dir, exist_ok=True)
+
+    # 5. Build wget command
+    cmd = build_webserver_wget_command(url, cut_dirs=cut_dirs, dst_dir=dst_dir, force=force, verbose=verbose)
+
+    # 6. Run wget command
+    try:
+        subprocess.run(cmd, check=True)
+    except subprocess.CalledProcessError as e:
+        raise subprocess.CalledProcessError(
+            returncode=e.returncode,
+            cmd=e.cmd,
+            output=e.output,
+            stderr=e.stderr,
+        )
+
+
+def ensure_wget_available() -> None:
+    """Raise FileNotFoundError if 'wget' is not on the system PATH."""
+    if shutil.which("wget") is None:
+        raise FileNotFoundError("The WGET software was not found. Please install WGET or add it to PATH.")
+
+
+def ensure_trailing_slash(url: str) -> str:
+    """Return `url` guaranteed to end with a slash."""
+    return url if url.endswith("/") else url.rstrip("/") + "/"
+
+
+def compute_cut_dirs(url: str) -> int:
+    """Compute the wget cut_dirs value to download directly in `dst_dir`.
+
+    Given a URL ending with '/', compute the total number of path segments.
+    By returning len(segments), we strip away all of them—so that files
+    within that final directory land directly in `dst_dir` without creating
+    an extra subfolder.
+    """
+    parsed = urllib.parse.urlparse(url)
+    path = parsed.path.strip("/")  # remove leading/trailing '/'
+    segments = path.split("/") if path else []
+    return len(segments)
+
+
+def build_webserver_wget_command(url: str, cut_dirs: int, dst_dir: str, force: bool, verbose: bool) -> list[str]:
+    """Construct the wget command list for subprocess.run.
+
+    Notes
+    -----
+    The following wget arguments are used
+      - -q         : quiet mode (no detailed progress)
+      - -r         : recursive
+      - -np        : no parent
+      - -nH        : no host directories
+      - --timestamping: download missing files or when remote version is newer
+      - --cut-dirs : strip all but the last path segment from the remote path
+      - -P dst_dir : download into `dst_dir`
+      - url
+    """
+    cmd = ["wget"]
+    if verbose:
+        cmd.append("-q")
+    cmd += [
+        "-r",
+        "-np",
+        "-nH",
+        f"--cut-dirs={cut_dirs}",
+    ]
+    if force:
+        cmd.append("--timestamping")  # -N
+
+    # Define source and destination directory
+    cmd += [
+        "-P",
+        dst_dir,
+        url,
+    ]
+    return cmd
+
+
+####--------------------------------------------------------------------.
+#### Download from Zenodo
+
+
+def download_zenodo_zip_file(url, dst_dir, force):
+    """Download zip file from zenodo and extract station raw data."""
+    # Download zip file
+    zip_filepath = _download_file_from_url(url, dst_dir=dst_dir, force=force)
+    # Extract the stations files from the downloaded station.zip file
+    _extract_station_files(zip_filepath, station_dir=dst_dir)
 
 
 def _download_file_from_url(url: str, dst_dir: str, force: bool = False) -> str:

{disdrodb-0.1.1 → disdrodb-0.1.2}/disdrodb/l0/configs/LPM/bins_diameter.yml

@@ -20,7 +20,7 @@ center:
   18: 6.75
   19: 7.25
   20: 7.75
-  21: 54
+  21: 9
 bounds:
   0:
     - 0.125
@@ -87,7 +87,7 @@ bounds:
     - 8.0
   21:
     - 8.0
-    - 100
+    - 10.0
 width:
   0: 0.125
   1: 0.125
@@ -110,4 +110,4 @@ width:
   18: 0.5
   19: 0.5
   20: 0.5
-  21: 92
+  21: 2

{disdrodb-0.1.1 → disdrodb-0.1.2}/disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml

@@ -37,3 +37,7 @@ list_particles: "str"
 raw_drop_concentration: "str"
 raw_drop_average_velocity: "str"
 raw_drop_number: "str"
+air_temperature: "float32"
+relative_humidity: "float32"
+wind_speed: "float32"
+wind_direction: "float32"

{disdrodb-0.1.1 → disdrodb-0.1.2}/disdrodb/l0/configs/PARSIVEL2/l0b_cf_attrs.yml

@@ -158,3 +158,19 @@ raw_drop_number:
   description: Drop counts per diameter and velocity class
   long_name: Raw drop number
   units: ""
+air_temperature:
+  description: "Air temperature in degrees Celsius (C)"
+  long_name: Air temperature
+  units: "C"
+relative_humidity:
+  description: "Relative humidity in percent (%)"
+  long_name: Relative humidity
+  units: "%"
+wind_speed:
+  description: "Wind speed in m/s"
+  long_name: Wind speed
+  units: "m/s"
+wind_direction:
+  description: "Wind direction in degrees (0-360)"
+  long_name: Wind direction
+  units: "degrees"

{disdrodb-0.1.1 → disdrodb-0.1.2}/disdrodb/l0/configs/PARSIVEL2/l0b_encodings.yml

@@ -331,3 +331,44 @@ raw_drop_number:
     - 5000
     - 32
     - 32
+air_temperature:
+  dtype: uint16
+  scale_factor: 0.1
+  add_offset: -99.9
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 5000
+relative_humidity:
+  dtype: uint16
+  scale_factor: 0.01
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 5000
+wind_speed:
+  dtype: uint16
+  scale_factor: 0.1
+  add_offset: -99.9
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 5000
+wind_direction:
+  dtype: uint16
+  zlib: true
+  complevel: 3
+  shuffle: true
+  fletcher32: false
+  contiguous: false
+  _FillValue: 65535
+  chunksizes: 5000

{disdrodb-0.1.1 → disdrodb-0.1.2}/disdrodb/l0/configs/PARSIVEL2/raw_data_format.yml

@@ -379,3 +379,43 @@ raw_drop_number:
     - diameter_bin_center
   n_values: 1024
   field_number: "93"
+air_temperature:
+  n_digits: 4
+  n_characters: 5
+  n_decimals: 1
+  n_naturals: 2
+  data_range:
+    - -40
+    - 70
+  nan_flags: 99999
+  field_number: "521"
+relative_humidity:
+  n_digits: 5
+  n_characters: 5
+  n_decimals: 0
+  n_naturals: 5
+  data_range:
+    - 0
+    - 99999
+  nan_flags: 99999
+  field_number: "522"
+wind_speed:
+  n_digits: 3
+  n_characters: 4
+  n_decimals: 1
+  n_naturals: 2
+  data_range:
+    - 0
+    - 60
+  nan_flags: null
+  field_number: "523"
+wind_direction:
+  n_digits: 3
+  n_characters: 3
+  n_decimals: 0
+  n_naturals: 3
+  data_range:
+    - 0
+    - 360
+  nan_flags: 999
+  field_number: "524"

disdrodb-0.1.2/disdrodb/l0/readers/PARSIVEL/GPM/PIERS.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+import pandas as pd
+
+from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
+from disdrodb.l0.l0a_processing import read_raw_text_file
+
+
+@is_documented_by(reader_generic_docstring)
+def reader(
+    filepath,
+    logger=None,
+):
+    """Reader."""
+    ##------------------------------------------------------------------------.
+    #### Define column names
+    column_names = ["time", "TO_BE_SPLITTED"]
+
+    ##------------------------------------------------------------------------.
+    #### Define reader options
+    reader_kwargs = {}
+    # - Define delimiter
+    reader_kwargs["delimiter"] = ";"
+    # - Skip first row as columns names
+    reader_kwargs["header"] = None
+    # Skip first row as columns names
+    reader_kwargs["header"] = None
+    reader_kwargs["skiprows"] = 0
+    # - Skip file with encoding errors
+    reader_kwargs["encoding_errors"] = "ignore"
+    # - Avoid first column to become df index !!!
+    reader_kwargs["index_col"] = False
+    # - Define behaviour when encountering bad lines
+    reader_kwargs["on_bad_lines"] = "skip"
+    # - Define reader engine
+    #   - C engine is faster
+    #   - Python engine is more feature-complete
+    reader_kwargs["engine"] = "python"
+    # - Define on-the-fly decompression of on-disk data
+    #   - Available: gzip, bz2, zip
+    reader_kwargs["compression"] = "infer"
+    # - Strings to recognize as NA/NaN and replace with standard NA flags
+    #   - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
+    #                       '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
+    #                       'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
+    reader_kwargs["na_values"] = ["na", "", "error", "NA", "-.-"]
+
+    ##------------------------------------------------------------------------.
+    #### Read the data
+    df = read_raw_text_file(
+        filepath=filepath,
+        column_names=column_names,
+        reader_kwargs=reader_kwargs,
+        logger=logger,
+    )
+
+    ##------------------------------------------------------------------------.
+    #### Adapt the dataframe to adhere to DISDRODB L0 standards
+    # Convert time column to datetime
+    df_time = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
+
+    # Split the 'TO_BE_SPLITTED' column
+    df = df["TO_BE_SPLITTED"].str.split(",", expand=True, n=9)
+
+    # Assign column names
+    columns_names = [
+        "station_name",
+        "sensor_status",
+        "sensor_temperature",
+        "number_particles",
+        "rainfall_rate_32bit",
+        "reflectivity_16bit",
+        "mor_visibility",
+        "weather_code_synop_4680",
+        "weather_code_synop_4677",
+        "raw_drop_number",
+    ]
+    df.columns = columns_names
+
+    # Add the time column
+    df["time"] = df_time
+
+    # Drop columns not agreeing with DISDRODB L0 standards
+    df = df.drop(columns=["station_name"])
+
+    # Drop rows with invalid values
+    # --> Ensure that weather_code_synop_4677 has length 2
+    # --> If a previous column is missing it will have 000
+    df = df[df["weather_code_synop_4677"].str.len() == 2]
+
+    # Return the dataframe adhering to DISDRODB L0 standards
+    return df

disdrodb-0.1.2/disdrodb/l0/readers/PARSIVEL/JAPAN/JMA.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+import pandas as pd
+
+from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
+from disdrodb.l0.l0a_processing import read_raw_text_file
+
+
+@is_documented_by(reader_generic_docstring)
+def reader(
+    filepath,
+    logger=None,
+):
+    """Reader."""
+    ##------------------------------------------------------------------------.
+    #### Define column names
+    column_names = ["TO_SPLIT"]
+
+    ##------------------------------------------------------------------------.
+    #### Define reader options
+    reader_kwargs = {}
+
+    # - Define delimiter
+    reader_kwargs["delimiter"] = "\\n"
+
+    # - Skip first row as columns names
+    reader_kwargs["header"] = None
+
+    # - Skip header
+    reader_kwargs["skiprows"] = 0
+
+    # - Define encoding
+    reader_kwargs["encoding"] = "ISO-8859-1"
+
+    # - Avoid first column to become df index !!!
+    reader_kwargs["index_col"] = False
+
+    # - Define behaviour when encountering bad lines
+    reader_kwargs["on_bad_lines"] = "skip"
+
+    # - Define reader engine
+    #   - C engine is faster
+    #   - Python engine is more feature-complete
+    reader_kwargs["engine"] = "python"
+
+    # - Define on-the-fly decompression of on-disk data
+    #   - Available: gzip, bz2, zip
+    # reader_kwargs['compression'] = 'xz'
+
+    # - Strings to recognize as NA/NaN and replace with standard NA flags
+    #   - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
+    #                       '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
+    #                       'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
+    reader_kwargs["na_values"] = ["na", "error", "-.-", " NA"]
+
+    ##------------------------------------------------------------------------.
+    #### Read the data
+    df = read_raw_text_file(
+        filepath=filepath,
+        column_names=column_names,
+        reader_kwargs=reader_kwargs,
+        logger=logger,
+    )
+
+    ##------------------------------------------------------------------------.
+    #### Adapt the dataframe to adhere to DISDRODB L0 standards
+    # Remove rows with less than 97 characters (empty spectrum --> 97 characters)
+    df = df[df["TO_SPLIT"].str.len() >= 97]
+
+    # Split into columns and assign name
+    df = df["TO_SPLIT"].str.split(";", expand=True, n=14)
+    columns = [
+        "date",
+        "time",
+        "rainfall_rate_32bit",
+        "rainfall_accumulated_32bit",
+        "weather_code_synop_4680",
+        "weather_code_metar_4678",
+        "weather_code_nws",
+        "reflectivity_32bit",
+        "mor_visibility",
+        "laser_amplitude",
+        "number_particles",
+        "sensor_temperature",
+        "sensor_heating_current",
+        "sensor_battery_voltage",
+        "raw_drop_number",
+    ]
+    df.columns = columns
+
+    # Add datetime time column
+    df["time"] = df["date"] + "-" + df["time"]
+    df["time"] = pd.to_datetime(df["time"], format="%Y/%m/%d-%H:%M:%S", errors="coerce")
+    df = df.drop(columns=["date"])
+
+    # Preprocess the raw spectrum
+    # - The '<SPECTRUM>ZERO</SPECTRUM>' indicates no drops detected
+    # --> "" generates an array of zeros in L0B processing
+    df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>ZERO</SPECTRUM>", "")
+
+    # Remove <SPECTRUM> and </SPECTRUM>" acronyms from the raw_drop_number field
+    df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
+    df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
+
+    # Add 0 before every , if , not preceded by a digit
+    # Example: ',,1,,' --> '0,0,1,0,'
+    df["raw_drop_number"] = df["raw_drop_number"].str.replace(r"(?<!\d);", "0;", regex=True)
+
+    # Return the dataframe adhering to DISDRODB L0 standards
+    return df