disdrodb 0.2.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

disdrodb/cli/disdrodb_run_station.py

@@ -0,0 +1,184 @@
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2026 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Script to launch all DISDRODB products generation for a given station."""
+
+import sys
+
+import click
+
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
+    click_l0_archive_options,
+    click_metadata_archive_dir_option,
+    click_processing_options,
+    click_station_arguments,
+    parse_archive_dir,
+)
+
+sys.tracebacklimit = 0  # avoid full traceback error if occur
+
+
+# -------------------------------------------------------------------------.
+# Click Command Line Interface decorator
+@click.command()
+@click_station_arguments
+@click_processing_options
+@click_l0_archive_options
+@click.option(
+    "-l1",
+    "--l1_processing",
+    type=bool,
+    show_default=True,
+    default=True,
+    help="Run L1 processing",
+)
+@click.option(
+    "-l2e",
+    "--l2e_processing",
+    type=bool,
+    show_default=True,
+    default=True,
+    help="Run L2E processing",
+)
+@click.option(
+    "-l2m",
+    "--l2m_processing",
+    type=bool,
+    show_default=True,
+    default=True,
+    help="Run L2M processing.",
+)
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
+def disdrodb_run_station(
+    # Station arguments
+    data_source: str,
+    campaign_name: str,
+    station_name: str,
+    # L0 archive options
+    l0a_processing: bool = True,
+    l0b_processing: bool = True,
+    l0c_processing: bool = True,
+    remove_l0a: bool = False,
+    remove_l0b: bool = False,
+    # Higher level processing options
+    l1_processing: bool = True,
+    l2e_processing: bool = True,
+    l2m_processing: bool = True,
+    # Processing options
+    force: bool = False,
+    verbose: bool = True,
+    parallel: bool = True,
+    debugging_mode: bool = False,
+    # DISDRODB root directories
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
+):
+    """Run the complete processing of a specific DISDRODB station.
+
+    This function processes a single station through the complete DISDRODB processing
+    chain, from raw data ingestion to final derived products. All processing levels
+    are executed in sequence for the specified station.
+
+    \b
+    Processing Levels:
+        L0A: Raw data converted to DISDRODB standardized Apache Parquet format
+        L0B: L0A data converted to DISDRODB standardized netCDF4 format
+        L0C: Apply time QC and consolidate L0B data into (by default daily) netCDF files
+        L1:  Temporally resample L0C data to standard intervals (1-60 minutes), apply QC algorithms
+                and determine precipitation phase and hydrometeors types
+        L2E: Compute empirical integral DSD variables (e.g., rain rate, liquid water content)
+        L2M: Fit parametric DSD models and compute model-based integral DSD variables
+
+    \b
+    Station Specification:
+        Requires exact specification of data_source, campaign_name, and station_name.
+        All three parameters must be provided and are case-sensitive (UPPER CASE required).
+
+    \b
+    Performance Options:
+        --parallel: Uses multiple processes for faster processing (default: True)
+        If parallel processing is enabled, each process will use a single thread
+        to avoid issues with the HDF/netCDF library.
+        The DASK_NUM_WORKERS environment variable controls the number of processes
+        to use.A sensible default is automatically set by the software.
+        --debugging_mode: Processes only a subset of data for testing
+        --force: Overwrites existing output files (default: False)
+
+
+    \b
+    Examples:
+        # Process a single station with full processing chain
+        disdrodb_run_station EPFL HYMEX_LTE_SOP2 10
+
+        # Process station, skip L0A processing (start from existing L0B data)
+        disdrodb_run_station NASA IFLOODS apu01 --l0a_processing False
+
+        # Process station with debugging mode and custom workers
+        DASK_NUM_WORKERS=4 disdrodb_run_station NETHERLANDS DELFT PAR001_Cabauw --debugging_mode True
+
+        # Process station, skip final L2M level
+        disdrodb_run_station FRANCE ENPC_CARNOT Carnot_Pars1 --l2m_processing False
+
+        # Force overwrite existing files with verbose output
+        disdrodb_run_station EPFL HYMEX_2012 10 --force True --verbose True
+
+    \b
+    Data Management:
+        --remove_l0a: Delete L0A files after L0B processing (saves disk space)
+        --remove_l0b: Delete L0B files after L0C processing (saves disk space)
+        Use with caution - removed files cannot be recovered without reprocessing
+
+    \b
+    Important Notes:
+        - Data source, campaign, and station names must be UPPER CASE
+        - All three station identifiers are required (no wildcards or filtering)
+        - Processing chain: L0A → L0B → L0C → L1 → L2E → L2M
+        - You can skip early or late processing levels, but not intermediate ones
+        - Processing validates chain consistency and will raise errors for gaps
+        - Station-specific processing may require significant time for large datasets
+        - Use --debugging_mode for initial testing with reduced data volumes
+    """  # noqa: D301
+    from disdrodb.routines import run_station
+
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+    metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
+
+    run_station(
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
+        # Station arguments
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+        # L0 archive options
+        l0a_processing=l0a_processing,
+        l0b_processing=l0b_processing,
+        l0c_processing=l0c_processing,
+        remove_l0a=remove_l0a,
+        remove_l0b=remove_l0b,
+        # Higher level processing options
+        l1_processing=l1_processing,
+        l2e_processing=l2e_processing,
+        l2m_processing=l2m_processing,
+        # Processing options
+        force=force,
+        verbose=verbose,
+        debugging_mode=debugging_mode,
+        parallel=parallel,
+    )

disdrodb/cli/disdrodb_upload_archive.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,7 +17,6 @@
 """Routines to upload data to the DISDRODB Decentralized Data Archive."""
 
 import sys
-from typing import Optional
 
 import click
 
@@ -41,48 +38,60 @@ sys.tracebacklimit = 0  # avoid full traceback error if occur
 @click_metadata_archive_dir_option
 def disdrodb_upload_archive(
     # DISDRODB root directories
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
     # Stations options
-    data_sources: Optional[str] = None,
-    campaign_names: Optional[str] = None,
-    station_names: Optional[str] = None,
+    data_sources: str | None = None,
+    campaign_names: str | None = None,
+    station_names: str | None = None,
     # Upload options
-    platform: Optional[str] = None,
+    platform: str | None = None,
     force: bool = False,
 ):
-    """Find all stations containing local data and upload them to a remote repository.
-
-    Parameters
-    ----------
-    platform: str, optional
-        Name of the remote platform.
-        The default platform is ``"sandbox.zenodo"`` (for testing purposes).
-        Switch to ``"zenodo"`` for final data dissemination.
-    force: bool, optional
-        If ``True``, upload even if data already exists on another remote location.
-        The default value is ``force=False``.
-    data_archive_dir : str (optional)
-        The directory path where the DISDRODB Data Archive is located.
-        The directory path must end with ``<...>/DISDRODB``.
-        If ``None``, it uses the ``data_archive_dir`` path specified
-        in the DISDRODB active configuration.
-
-    Other Parameters
-    ----------------
-    data_sources: str or list of str, optional
-        Data source name (eg: EPFL).
-        If not provided (``None``), all data sources will be uploaded.
-        The default value is ``data_source=None``.
-    campaign_names: str or list of str, optional
-        Campaign name (eg:  EPFL_ROOF_2012).
-        If not provided (``None``), all campaigns will be uploaded.
-        The default value is ``campaign_name=None``.
-    station_names: str or list of str, optional
-        Station name.
-        If not provided (``None``), all stations will be uploaded.
-        The default value is ``station_name=None``.
-    """
+    """Upload raw data for multiple DISDRODB stations to the DISDRODB Decentralized Data Archive.
+
+    Currently, only upload to the Zenodo data repository is implemented.
+    The station metadata files are automatically updated with the remote data URL.
+
+    PLEASE UPLOAD JUST YOUR DATA !
+
+    \b
+    Station Selection:
+        If no station filters are specified, uploads ALL stations with local data.
+        Use data_sources, campaign_names, and station_names to filter stations.
+        Filters work together to narrow down the selection (AND logic).
+        Only stations with local raw data files will be uploaded.
+
+    \b
+    Upload Behavior:
+        By default, upload is skipped if data already exists on a remote location.
+        Use '--force True' to upload even if data already exists remotely.
+        Warning: Forcing upload may create duplicate versions on the remote platform.
+
+    \b
+    Upload Options:
+        --platform: Remote platform name (default: 'sandbox.zenodo' for testing)
+        Use 'zenodo' for final data dissemination.
+        --force: Upload even if data already exists remotely (default: False)
+
+    \b
+    Archive Directories:
+        --data_archive_dir: Custom path to DISDRODB data archive
+        --metadata_archive_dir: Custom path to DISDRODB metadata archive
+        If not specified, paths from the active DISDRODB configuration are used
+
+    \b
+    Examples:
+
+        # Upload a specific data sources to Zenodo Sandbox
+        disdrodb_upload_archive --data_sources 'YOUR_DATA_SOURCE' --platform sandbox.zenodo
+
+        # Upload a specific data sources to Zenodo and force upload
+        disdrodb_upload_archive --data_sources 'YOUR_DATA_SOURCE' --platform zenodo --force True
+
+        # Upload specific campaigns to Zenodo
+        disdrodb_upload_archive --campaign_names 'HYMEX_LTE_SOP2 IFLOODS' --platform zenodo
+    """  # noqa: D301
     from disdrodb.data_transfer.upload_data import upload_archive
 
     data_archive_dir = parse_archive_dir(data_archive_dir)

disdrodb/cli/disdrodb_upload_station.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,7 +17,6 @@
 """Routines to upload station data to the DISDRODB Decentralized Data Archive."""
 
 import sys
-from typing import Optional
 
 import click
 
@@ -45,41 +42,50 @@ def disdrodb_upload_station(
     campaign_name: str,
     station_name: str,
     # Upload options
-    platform: Optional[str] = None,
+    platform: str | None = None,
     force: bool = False,
     # DISDRODB root directories
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
 ):
-    """
-    Upload data from a single DISDRODB station on a remote repository.
-
-    This function also automatically update the disdrodb_data url in the metadata file.
-
-    Parameters
-    ----------
-    data_source : str
-        The name of the institution (for campaigns spanning multiple countries) or
-        the name of the country (for campaigns or sensor networks within a single country).
-        Must be provided in UPPER CASE.
-    campaign_name : str
-        The name of the campaign. Must be provided in UPPER CASE.
-    station_name : str
-        The name of the station.
-    data_archive_dir : str (optional)
-        The directory path where the DISDRODB Data Archive is located.
-        The directory path must end with ``<...>/DISDRODB``.
-        If ``None``, it uses the ``data_archive_dir`` path specified
-        in the DISDRODB active configuration.
-    platform: str, optional
-        Name of the remote data storage platform.
-        The default platform is ``"sandbox.zenodo"`` (for testing purposes).
-        Switch to ``"zenodo"`` for final data dissemination.
-    force: bool, optional
-        If ``True``, upload the data and overwrite the ``disdrodb_data_url``.
-        The default value is ``force=False``.
-
-    """
+    """Upload raw data from a single DISDRODB station to the DISDRODB Decentralized Data Archive.
+
+    Currently, only upload to the Zenodo data repository is implemented.
+
+    The station metadata file is automatically updated with the remote data URL.
+
+    PLEASE UPLOAD JUST YOUR DATA !
+
+
+    \b
+    Upload Behavior:
+        By default, upload is skipped if data already exists on a remote location.
+        Use '--force True' to upload even if data already exists remotely.
+        Warning: Forcing upload will overwrite the existing disdrodb_data_url in metadata.
+
+    \b
+    Upload Options:
+        --platform: Remote platform name (default: 'sandbox.zenodo' for testing)
+            Use 'zenodo' for final data dissemination
+        --force: Upload even if data already exists remotely (default: False)
+
+    \b
+    Archive Directories:
+        --data_archive_dir: Custom path to DISDRODB data archive
+        --metadata_archive_dir: Custom path to DISDRODB metadata archive
+        If not specified, paths from the active DISDRODB configuration are used
+
+    \b
+    Examples:
+        # Upload a station to sandbox for testing
+        disdrodb_upload_station DATA_SOURCE CAMPAIGN_NAME STATION_NAME
+
+        # Upload a station to production Zenodo
+        disdrodb_upload_station DATA_SOURCE CAMPAIGN_NAME STATION_NAME --platform zenodo
+
+        # Force upload and update metadata URL
+        disdrodb_upload_station DATA_SOURCE CAMPAIGN_NAME STATION_NAME --force True
+    """  # noqa: D301
     from disdrodb.data_transfer.upload_data import upload_station
 
     data_archive_dir = parse_archive_dir(data_archive_dir)

disdrodb/configs.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,7 +18,6 @@
 
 import os
 import shutil
-from typing import Optional
 
 from disdrodb.utils.yaml import read_yaml, write_yaml
 
@@ -34,13 +31,13 @@ def _define_config_filepath():
 
 
 def define_configs(
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
-    scattering_table_dir: Optional[str] = None,
-    products_configs_dir: Optional[str] = None,
-    folder_partitioning: Optional[str] = None,
-    zenodo_token: Optional[str] = None,
-    zenodo_sandbox_token: Optional[str] = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
+    scattering_table_dir: str | None = None,
+    products_configs_dir: str | None = None,
+    folder_partitioning: str | None = None,
+    zenodo_token: str | None = None,
+    zenodo_sandbox_token: str | None = None,
 ):
     """
     Defines the DISDRODB configuration file with the given credentials and base directory.
@@ -194,7 +191,11 @@ def get_scattering_table_dir(scattering_table_dir=None):
     if scattering_table_dir is None:
         scattering_table_dir = disdrodb.config.get("scattering_table_dir", None)
     if scattering_table_dir is None:
-        raise ValueError("The directory where to save DISDRODB T-Matrix scattering tables is not specified.")
+        msg = (
+            "The directory where to save DISDRODB T-Matrix scattering look-up tables is not specified. "
+            + "Please specify it using disdrodb.define_configs(scattering_table_dir=<path>)"
+        )
+        raise ValueError(msg)
     scattering_table_dir = check_scattering_table_dir(scattering_table_dir)  # ensure Path converted to str
     return scattering_table_dir
 
@@ -246,16 +247,19 @@ def get_default_products_configs_dir():
     return products_configs_dir
 
 
-def get_products_configs_dir():
+def get_products_configs_dir(products_configs_dir=None):
     """Return the DISDRODB products configuration directory."""
     import disdrodb
 
-    if os.environ.get("PYTEST_CURRENT_TEST"):
+    if products_configs_dir is not None:
+        return str(products_configs_dir)
+
+    if os.environ.get("PYTEST_CURRENT_TEST") and not os.environ.get("DISDRODB_VALIDATION_FLAG"):
         products_configs_dir = os.path.join(disdrodb.package_dir, "tests", "products")
     else:
         products_configs_dir = disdrodb.config.get("products_configs_dir", None)
-        if products_configs_dir is None:
-            products_configs_dir = get_default_products_configs_dir()
+        if products_configs_dir is None:  # not specified by user
+            products_configs_dir = get_default_products_configs_dir()  # take disdrodb defaults
     return products_configs_dir
 
 

disdrodb/constants.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """DISDRODB constants."""
+
 import importlib
 
 ARCHIVE_VERSION = "V0"
@@ -41,7 +42,9 @@ COORDINATES = [
     "time",
     "sample_interval",
 ]
-OPTICAL_SENSORS = ["PARSIVEL", "PARSIVEL2", "LPM", "LPM_V0", "PWS100", "SWS250"]
+METEOROLOGICAL_VARIABLES = ["air_temperature", "relative_humidity", "wind_speed", "wind_direction"]
+
+OPTICAL_SENSORS = ["PARSIVEL", "PARSIVEL2", "LPM", "LPM_V0", "PWS100", "SWS250", "ODM470"]
 IMPACT_SENSORS = ["RD80"]
 
 PRODUCTS = ["RAW", "L0A", "L0B", "L0C", "L1", "L2E", "L2M"]

disdrodb/data_transfer/__init__.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by