disdrodb 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

disdrodb/__init__.py

@@ -78,6 +78,8 @@ from disdrodb.routines import (
     run_l2e_station,
     run_l2m,
     run_l2m_station,
+    run,
+    run_station,
 )
 from disdrodb.utils.manipulations import convert_from_decibel as idecibel
 from disdrodb.utils.manipulations import convert_to_decibel as decibel
@@ -158,6 +160,7 @@ __all__ = [
     "read_metadata_archive",
     "read_station_metadata",
     # Functions invoking the disdrodb_run_* scripts in the terminals
+    "run",
     "run_l0",
     "run_l0_station",
     "run_l0a",
@@ -172,6 +175,7 @@ __all__ = [
     "run_l2e_station",
     "run_l2m",
     "run_l2m_station",
+    "run_station",
 ]
 
 

disdrodb/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.4.0'
-__version_tuple__ = version_tuple = (0, 4, 0)
+__version__ = version = '0.5.1'
+__version_tuple__ = version_tuple = (0, 5, 1)
 
 __commit_id__ = commit_id = None

disdrodb/accessor/methods.py

@@ -135,6 +135,20 @@ class DISDRODB_Dataset_Accessor(DISDRODB_Base_Accessor):
 
         return plot_raw_and_filtered_spectra(self._obj, **kwargs)
 
+    @property
+    def psd(self):
+        """Return PSD class from DISDRODB L2M product."""
+        from disdrodb.psd.models import create_psd_from_dataset
+
+        return create_psd_from_dataset(self._obj)
+
+    @property
+    def psd_parameters(self):
+        """Return PSD parameters from DISDRODB L2M product."""
+        from disdrodb.psd.models import get_parameters_from_dataset
+
+        return get_parameters_from_dataset(self._obj)
+
 
 @xr.register_dataarray_accessor("disdrodb")
 class DISDRODB_DataArray_Accessor(DISDRODB_Base_Accessor):

disdrodb/api/checks.py

@@ -171,16 +171,17 @@ def check_folder_partitioning(folder_partitioning):
     folder_partitioning : str or None
         Defines the subdirectory structure based on the dataset's start time.
         Allowed values are:
-          - "" or None: No additional subdirectories, files are saved directly in dir.
-          - "year": Files are stored under a subdirectory for the year (<dir>/2025).
-          - "year/month": Files are stored under subdirectories by year and month (<dir>/2025/04).
-          - "year/month/day": Files are stored under subdirectories by year, month and day (<dir>/2025/04/01).
-          - "year/month_name": Files are stored under subdirectories by year and month name (<dir>/2025/April).
-          - "year/quarter": Files are stored under subdirectories by year and quarter (<dir>/2025/Q2).
+
+        - "" or None: No additional subdirectories, files are saved directly in dir.
+        - "year": Files are stored under a subdirectory for the year (<dir>/2025).
+        - "year/month": Files are stored under subdirectories by year and month (<dir>/2025/04).
+        - "year/month/day": Files are stored under subdirectories by year, month and day (<dir>/2025/04/01).
+        - "year/month_name": Files are stored under subdirectories by year and month name (<dir>/2025/April).
+        - "year/quarter": Files are stored under subdirectories by year and quarter (<dir>/2025/Q2).
 
     Returns
     -------
-    folder_partitioning
+    str
         The verified folder partitioning scheme.
     """
     valid_options = ["", "year", "year/month", "year/month/day", "year/month_name", "year/quarter"]

disdrodb/api/io.py

@@ -335,24 +335,24 @@ def filter_dataset_by_time(ds, start_time=None, end_time=None):
 
     Parameters
     ----------
-    ds : xr.Dataset
+    ds : xarray.Dataset
         Dataset with a `time` coordinate.
-    start_time : np.datetime64 or None
+    start_time : str, numpy.datetime64 or None
         Inclusive start bound. If None, no lower bound is applied.
-    end_time : np.datetime64 or None
+    end_time : str, numpy.datetime64 or None
         Inclusive end bound. If None, no upper bound is applied.
 
     Returns
     -------
-    xr.Dataset
+    xarray.Dataset
         Subset dataset with the same ordering of timesteps (duplicates preserved).
     """
     time = ds["time"].to_numpy()
     mask = np.ones(time.shape, dtype=bool)
     if start_time is not None:
-        mask &= time >= np.array(start_time, dtype="datetime64[ns]")
+        mask &= time >= np.array(check_time(start_time), dtype="datetime64[ns]")
     if end_time is not None:
-        mask &= time <= np.array(end_time, dtype="datetime64[ns]")
+        mask &= time <= np.array(check_time(end_time), dtype="datetime64[ns]")
     return ds.isel(time=np.where(mask)[0])
 
 
@@ -445,30 +445,73 @@ def open_netcdf_files(
     engine="netcdf4",
     **open_kwargs,
 ):
-    """Open DISDRODB netCDF files using xarray.
-
-    Using data_vars="minimal", coords="minimal", compat="override"
-    --> will only concatenate those variables with the time dimension,
-    --> will skip any checking for variables that don't have a time dimension
-       (simply pick the variable from the first file).
-    https://github.com/pydata/xarray/issues/1385#issuecomment-1958761334
-
-    Using combine="nested" and join="outer" ensure that duplicated timesteps
-    are not overwritten!
-
-    When decode_cf=False
-    --> lat,lon are data_vars and get concatenated without any checking or reading
-    When decode_cf=True
-    --> lat, lon are promoted to coords, then get checked for equality across all files
-
-    For L0B product, if sample_interval variable is present and varies with time,
-    this function concatenate the variable over time without problems.
-    For L0C product, if sample_interval changes across listed files,
-    only sample_interval of first file is reported.
-    --> open_dataset take care of just providing filepaths of files with same sample interval.
-    In L1 and L2 processing, only filepaths of files with same sample interval
-    must be passed to this function.
+    """Open DISDRODB NetCDF files using xarray.
+
+    This function opens and concatenates multiple NetCDF files using
+    ``xarray.open_mfdataset`` with settings optimized for time-based
+    concatenation and minimal variable checking.
+
+    The function uses ``data_vars="minimal"``, ``coords="minimal"``,
+    and ``compat="override"`` to:
+
+    - Concatenate only variables that depend on the time dimension.
+    - Skip consistency checks for variables without a time dimension, taking them from the first file instead.
+
+    See: https://github.com/pydata/xarray/issues/1385#issuecomment-1958761334
+
+    Using ``combine="nested"`` and ``join="outer"`` ensures that duplicated
+    timesteps are preserved and not overwritten.
+
+    Behavior depends on ``decode_cf``:
+
+    - If ``decode_cf=False``: ``lat`` and ``lon`` are treated as data variables and concatenated without validation.
+    - If ``decode_cf=True``: ``lat`` and ``lon`` are promoted to coordinates and checked for  equality across files.
+
+    Special handling of ``sample_interval``:
+
+    - For L0B products, if ``sample_interval`` varies with time, it is safely concatenated.
+    - For L0C products, if ``sample_interval`` differs across files, only the value from the first file is retained.
+    - For L1 and L2 processing, only files with identical ``sample_interval`` values should be passed to this function.
+
+    Parameters
+    ----------
+    filepaths : str or sequence of str
+        Path(s) to NetCDF files to open.
+    chunks : int, dict, or None, optional
+        Chunking strategy passed to xarray for dask-backed arrays.
+        Use ``-1`` to load data into a single chunk (default).
+    start_time : str or datetime-like or None, optional
+        Start time for temporal subsetting.
+    end_time : str or datetime-like or None, optional
+        End time for temporal subsetting.
+    variables : sequence of str or None, optional
+        Subset of variables to retain.
+    parallel : bool, optional
+        Whether to open files in parallel using dask.
+        The default is ``False``.
+    compute : bool, optional
+        Whether to immediately compute the dataset when using dask.
+        The default is ``True``.
+    engine : str, optional
+        Backend engine used by xarray to read NetCDF files.
+        The default is "netcdf4".
+    **open_kwargs
+        Additional keyword arguments passed to
+        ``xarray.open_mfdataset``.
+
+    Returns
+    -------
+    xarray.Dataset
+        The opened and concatenated dataset.
 
+    See Also
+    --------
+    xarray.open_mfdataset
+
+    Notes
+    -----
+    This function is decorated with ``ensure_safe_open_mfdataset`` to
+    protect against unsafe or incompatible combinations of arguments.
     """
     import xarray as xr
 
@@ -804,6 +847,15 @@ def open_readers_directory():
     open_file_explorer(readers_directory)
 
 
+def open_products_options():
+    """Open the disdrodb product options directory."""
+    from disdrodb.configs import get_products_configs_dir
+
+    products_configs_dir = get_products_configs_dir()
+
+    open_file_explorer(products_configs_dir)
+
+
 def open_metadata_archive(
     metadata_archive_dir=None,
 ):

disdrodb/api/path.py

@@ -358,12 +358,13 @@ def define_partitioning_tree(time, folder_partitioning):
     folder_partitioning : str or None
         Define the subdirectory structure where saving files.
         Allowed values are:
-          - None: Files are saved directly in data_dir.
-          - "year": Files are saved under a subdirectory for the year.
-          - "year/month": Files are saved under subdirectories for year and month.
-          - "year/month/day": Files are saved under subdirectories for year, month and day
-          - "year/month_name": Files are stored under subdirectories by year and month name
-          - "year/quarter": Files are saved under subdirectories for year and quarter.
+
+        - None: Files are saved directly in data_dir.
+        - "year": Files are saved under a subdirectory for the year.
+        - "year/month": Files are saved under subdirectories for year and month.
+        - "year/month/day": Files are saved under subdirectories for year, month and day
+        - "year/month_name": Files are stored under subdirectories by year and month name
+        - "year/quarter": Files are saved under subdirectories for year and quarter.
 
     Returns
     -------
@@ -411,12 +412,14 @@ def define_file_folder_path(obj, dir_path, folder_partitioning):
     folder_partitioning : str or None
         Define the subdirectory structure where saving files.
         Allowed values are:
-          - None or "": Files are saved directly in data_dir.
-          - "year": Files are saved under a subdirectory for the year.
-          - "year/month": Files are saved under subdirectories for year and month.
-          - "year/month/day": Files are saved under subdirectories for year, month and day
-          - "year/month_name": Files are stored under subdirectories by year and month name
-          - "year/quarter": Files are saved under subdirectories for year and quarter.
+
+        - None or "": Files are saved directly in data_dir.
+        - "year": Files are saved under a subdirectory for the year.
+        - "year/month": Files are saved under subdirectories for year and month.
+        - "year/month/day": Files are saved under subdirectories for year, month and day
+        - "year/month_name": Files are stored under subdirectories by year and month name
+        - "year/quarter": Files are saved under subdirectories for year and quarter.
+
 
     Returns
     -------
@@ -682,10 +685,10 @@ def define_filename(
        Name of the campaign.
     station_name : str
        Name of the station.
-    start_time : datetime.datatime, optional
+    start_time : datetime.datetime, optional
         Start time.
         Required if add_time_period = True.
-    end_time : datetime.datatime, optional
+    end_time : datetime.datetime, optional
         End time.
         Required if add_time_period = True.
     temporal_resolution : str, optional

disdrodb/api/search.py

@@ -310,8 +310,7 @@ def available_stations(
     metadata_archive_dir=None,
     **filter_kwargs,
 ):
-    """
-    Return stations information for which metadata or product data are available on disk.
+    """Return stations information for which metadata or product data are available on disk.
 
     This function queries the DISDRODB Metadata Archive and, optionally, the
     local DISDRODB Data Archive to identify stations that satisfy the specified
@@ -334,31 +333,29 @@ def available_stations(
         it lists the stations present in the local DISDRODB Data Archive given the specified filtering criteria.
         The default is None.
 
-    data_sources : str or sequence of str, optional
+    data_sources : str or list of str, optional
         One or more data source identifiers to filter stations by.
         The name(s) must be UPPER CASE.
         If None, no filtering on data source is applied. The default is is ``None``.
-    campaign_names : str or sequence of str, optional
+    campaign_names : str or list of str, optional
         One or more campaign names to filter stations by.
         The name(s) must be UPPER CASE.
         If None, no filtering on campaign is applied. The default is is ``None``.
-    station_names : str or sequence of str, optional
+    station_names : str or list of str, optional
         One or more station names to include.
         If None, all stations matching other filters are considered. The default is is ``None``.
     available_data : bool, optional
 
         If ``product`` is not specified:
 
-            - if available_data is False, return stations present in the DISDRODB Metadata Archive
-            - if available_data is True, return stations with data available on the
-            online DISDRODB Decentralized Data Archive (i.e., stations with the disdrodb_data_url in the metadata).
+        - if ``available_data=False``, return stations present in the DISDRODB Metadata Archive.
+        - if ``available_data=True``, return stations with data available on the online DISDRODB Decentralized Data Archive (i.e., stations with the disdrodb_data_url in the metadata).
 
         If ``product`` is specified:
 
-            - if available_data is False, return stations where the product directory exists in the
-              in the local DISDRODB Data Archive
-            - if available_data is True, return stations where product data exists in the
-              in the local DISDRODB Data Archive.
+        - if ``available_data=False``, return stations where the product directory exists in the in the local DISDRODB Data Archive
+        - if ``available_data=True``, return stations where product data exists in the in the local DISDRODB Data Archive.
+
         The default is is False.
 
     return_tuple : bool, optional
@@ -368,17 +365,17 @@ def available_stations(
     raise_error_if_empty : bool, optional
         If True and no stations satisfy the criteria, raise a ``ValueError``.
         If False, return an empty list/tuple. The default is False.
-    invalid_fields_policy : {'raise', 'warn', 'ignore'}, optional
+    invalid_fields_policy : str, optional
         How to handle invalid filter values for ``data_sources``, ``campaign_names``,
         or ``station_names`` that are not present in the metadata archive:
 
-          - 'raise' : raise a ``ValueError`` (default)
-          - 'warn'  : emit a warning, then ignore invalid entries
-          - 'ignore': silently drop invalid entries
+        - 'raise' : raise a ``ValueError`` (default)
+        - 'warn'  : emit a warning, then ignore invalid entries
+        - 'ignore': silently drop invalid entries
 
     data_archive_dir : str or Path-like, optional
         Path to the root of the local DISDRODB Data Archive.
-        Required only if ``product``is specified.
+        Required only if ``product`` is specified.
         If None, the default data archive base directory is used. Default is None.
     metadata_archive_dir : str or Path-like, optional
         Path to the root of the DISDRODB Metadata Archive.
@@ -405,7 +402,7 @@ def available_stations(
     >>> raw_stations = available_stations(product="RAW", available_data=True)
     >>> # List stations of specific data sources
     >>> stations = available_stations(data_sources=["NASA", "EPFL"])
-    """
+    """  # noqa: E501
     # Retrieve DISDRODB Data and Metadata Archive directories
     metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
     product = check_product(product) if product is not None else None

disdrodb/cli/disdrodb_open_products_options.py

@@ -0,0 +1,38 @@
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2026 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Routine to open the DISDRODB Products Options."""
+
+import sys
+
+import click
+
+sys.tracebacklimit = 0  # avoid full traceback error if occur
+
+
+@click.command()
+def disdrodb_open_products_options():
+    """Open the DISDRODB Products Configuration Directory in the system file explorer.
+
+    Examples
+    --------
+        # Open the data archive from active configuration
+        disdrodb_open_product_options
+
+    """
+    from disdrodb.api.io import open_products_options
+
+    open_products_options()

disdrodb/cli/disdrodb_run.py

@@ -57,7 +57,7 @@ sys.tracebacklimit = 0  # avoid full traceback error if occur
     "--l2m_processing",
     type=bool,
     show_default=True,
-    default=True,
+    default=False,
     help="Run L2M processing.",
 )
 @click_processing_options
@@ -77,7 +77,7 @@ def disdrodb_run(
     # Higher level processing options
     l1_processing: bool = True,
     l2e_processing: bool = True,
-    l2m_processing: bool = True,
+    l2m_processing: bool = False,
     # Processing options
     force: bool = False,
     verbose: bool = True,

disdrodb/cli/disdrodb_run_station.py

@@ -59,7 +59,7 @@ sys.tracebacklimit = 0  # avoid full traceback error if occur
     "--l2m_processing",
     type=bool,
     show_default=True,
-    default=True,
+    default=False,
     help="Run L2M processing.",
 )
 @click_data_archive_dir_option
@@ -78,7 +78,7 @@ def disdrodb_run_station(
     # Higher level processing options
     l1_processing: bool = True,
     l2e_processing: bool = True,
-    l2m_processing: bool = True,
+    l2m_processing: bool = False,
     # Processing options
     force: bool = False,
     verbose: bool = True,
@@ -131,8 +131,8 @@ def disdrodb_run_station(
         # Process station with debugging mode and custom workers
         DASK_NUM_WORKERS=4 disdrodb_run_station NETHERLANDS DELFT PAR001_Cabauw --debugging_mode True
 
-        # Process station, skip final L2M level
-        disdrodb_run_station FRANCE ENPC_CARNOT Carnot_Pars1 --l2m_processing False
+        # Process station, create L2M product (disabled by default)
+        disdrodb_run_station FRANCE ENPC_CARNOT Carnot_Pars1 --l2m_processing True
 
         # Force overwrite existing files with verbose output
         disdrodb_run_station EPFL HYMEX_2012 10 --force True --verbose True

disdrodb/configs.py

@@ -282,7 +282,7 @@ def copy_default_products_configs(products_configs_dir):
 
     Returns
     -------
-    products_configs_dir
+    str
         The path to the newly created custom product configuration directory.
 
     """

disdrodb/data_transfer/download_data.py

@@ -24,6 +24,7 @@ import urllib.parse
 
 import click
 import pooch
+import requests
 import tqdm
 
 from disdrodb.api.path import define_metadata_filepath, define_station_dir
@@ -405,6 +406,8 @@ def build_webserver_wget_command(url: str, cut_dirs: int, dst_dir: str, verbose:
         f"--cut-dirs={cut_dirs}",
         # Downloads just new data without re-downloading existing files
         "--timestamping",  # -N
+        # Specify agent
+        "--user-agent=disdrodb (+https://github.com/ltelab/disdrodb)",
     ]
 
     # Define source and destination directory
@@ -541,15 +544,81 @@ def _download_file_from_url(url: str, dst_dir: str) -> str:
 
     os.makedirs(dst_dir, exist_ok=True)
 
+    # Check if it can be downloaded
+    if not is_programmatic_downloadable(url):
+        raise RuntimeError(
+            f"Cannot download data programmatically from '{url}' right now. "
+            "The server requires a web browser (e.g. WAF / anti-bot protection).",
+        )
+
     # Grab Pooch's logger and remember its current level
     logger = pooch.get_logger()
     orig_level = logger.level
     # Silence INFO messages (including the SHA256 print)
     logger.setLevel(logging.WARNING)
+
     # Define pooch downloader
-    downloader = pooch.HTTPDownloader(progressbar=True)
+    headers = {"User-Agent": "disdrodb (+https://github.com/ltelab/disdrodb)"}
+    # "Accept": "*/*"}
+    downloader = pooch.HTTPDownloader(progressbar=True, headers=headers)
     # Download the file
     pooch.retrieve(url=url, known_hash=None, path=dst_dir, fname=dst_filename, downloader=downloader, progressbar=tqdm)
     # Restore the previous logging level
     logger.setLevel(orig_level)
+
+    # Check file has been downloaded
+    # Note: `pooch.retrieve` will raise an exception if the download fails, so we do not
+    #       perform an additional explicit file existence/size check here.
+    # if not os.path.isfile(dst_filepath) or os.path.getsize(dst_filepath) == 0:
+    #     raise RuntimeError(f"URL {url} likely unreachable. Try manually.")
+
     return dst_filepath
+
+
+def is_programmatic_downloadable(url, timeout=5):
+    """Check whether a URL is programmatically downloadable.
+
+    WITHOUT downloading the file.
+
+    Returns
+    -------
+    bool
+        True  -> safe for pooch / requests
+        False -> browser-only / blocked
+    """
+    headers = {
+        # Range prevents full download (0-0 = 1 byte max)
+        "Range": "bytes=0-0",
+        # Explicitly non-browser
+        "User-Agent": "python-downloader/1.0",
+    }
+    # Open stream and close it after check terminated
+    try:
+        with requests.get(
+            url,
+            headers=headers,
+            allow_redirects=True,
+            timeout=timeout,
+            stream=True,
+        ) as r:
+
+            # --- Hard fail signals ---
+            if r.status_code in (202, 403, 401):
+                return False
+
+            if r.headers.get("x-amzn-waf-action") == "challenge":
+                return False
+
+            content_type = r.headers.get("Content-Type", "").lower()
+            if "text/html" in content_type:
+                return False
+
+            content_length = r.headers.get("Content-Length")
+            if content_length == "0":
+                return False
+
+            # If server honored Range, we are good
+            return r.status_code in (200, 206)
+
+    except requests.RequestException:
+        return False