disdrodb 0.2.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

disdrodb/__init__.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,6 +20,8 @@ import contextlib
 import os
 from importlib.metadata import PackageNotFoundError, version
 
+os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
+
 import disdrodb.accessor  # noqa
 from disdrodb._config import config  # noqa
 from disdrodb.api.configs import available_sensor_names

disdrodb/_config.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,6 +18,7 @@
 
 See https://donfig.readthedocs.io/en/latest/configuration.html for more info.
 """
+
 from donfig import Config
 
 from disdrodb.configs import read_configs

disdrodb/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.2.1'
-__version_tuple__ = version_tuple = (0, 2, 1)
+__version__ = version = '0.4.0'
+__version_tuple__ = version_tuple = (0, 4, 0)
 
 __commit_id__ = commit_id = None

disdrodb/accessor/__init__.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """This directory defines DISDRODB xarray accessors."""
+
 from .methods import DISDRODB_DataArray_Accessor, DISDRODB_Dataset_Accessor
 
 __all__ = ["DISDRODB_DataArray_Accessor", "DISDRODB_Dataset_Accessor"]

disdrodb/accessor/methods.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """This module defines DISDRODB xarray accessors."""
+
 import xarray as xr
 
 
@@ -32,7 +33,7 @@ class DISDRODB_Base_Accessor:
         from disdrodb.api.checks import check_time
 
         if "time" in self._obj.coords:
-            start_time = self._obj["time"].to_numpy()[0]
+            start_time = self._obj["time"].to_numpy().min()
         else:
             raise ValueError("Time coordinate not found")
         return check_time(start_time)
@@ -43,7 +44,7 @@ class DISDRODB_Base_Accessor:
         from disdrodb.api.checks import check_time
 
         if "time" in self._obj.coords:
-            end_time = self._obj["time"].to_numpy()[-1]
+            end_time = self._obj["time"].to_numpy().max()
         else:
             raise ValueError("Time coordinate not found")
         return check_time(end_time)
@@ -102,6 +103,12 @@ class DISDRODB_Base_Accessor:
 
         return plot_spectrum(self._obj, **kwargs)
 
+    def plot_nd(self, **kwargs):
+        """Plot drop number concentration N(D) timeseries."""
+        from disdrodb.viz.plots import plot_nd
+
+        return plot_nd(self._obj, **kwargs)
+
 
 @xr.register_dataset_accessor("disdrodb")
 class DISDRODB_Dataset_Accessor(DISDRODB_Base_Accessor):
@@ -122,12 +129,6 @@ class DISDRODB_Dataset_Accessor(DISDRODB_Base_Accessor):
         )
         return ds
 
-    def plot_nd(self, **kwargs):
-        """Plot drop number concentration N(D) timeseries."""
-        from disdrodb.viz.plots import plot_nd
-
-        return plot_nd(self._obj, **kwargs)
-
     def plot_raw_and_filtered_spectra(self, **kwargs):
         """Plot the raw and filtered spectra."""
         from disdrodb.viz.plots import plot_raw_and_filtered_spectra

disdrodb/api/checks.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,12 +15,12 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """DISDRODB Checks Functions."""
+
 import datetime
 import difflib
 import logging
 import os
 import re
-import sys
 import warnings
 
 import numpy as np
@@ -567,9 +565,7 @@ def check_filepaths(filepaths):
 
 def get_current_utc_time():
     """Get current UTC time."""
-    if sys.version_info >= (3, 11):
-        return datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
-    return datetime.datetime.utcnow()
+    return datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
 
 
 def check_start_end_time(start_time, end_time):

disdrodb/api/configs.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

disdrodb/api/create_directories.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -23,7 +21,6 @@
 
 import logging
 import os
-import shutil
 
 from disdrodb.api.checks import (
     check_campaign_name,
@@ -46,6 +43,7 @@ from disdrodb.api.path import (
 from disdrodb.configs import get_data_archive_dir, get_metadata_archive_dir
 from disdrodb.utils.directories import (
     create_directory,
+    remove_file_or_directories,
     remove_if_exists,
 )
 
@@ -59,7 +57,7 @@ def ensure_empty_data_dir(data_dir, force):
     """Remove the content of the data_dir directory."""
     # If force=True, remove all the directory content
     if force:
-        shutil.rmtree(data_dir)
+        remove_file_or_directories(data_dir)
         # Recreate the directory
         create_directory(data_dir)
     else:
@@ -258,7 +256,7 @@ def create_logs_directory(
 
     # Ensure empty log directory
     if os.path.isdir(logs_dir):
-        shutil.rmtree(logs_dir)
+        remove_file_or_directories(logs_dir)
 
     # Create logs directory
     os.makedirs(logs_dir, exist_ok=True)

disdrodb/api/info.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

disdrodb/api/io.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,22 +15,23 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Routines to list and open DISDRODB products."""
+
 import datetime
+import functools
 import os
-import shutil
 import subprocess
 import sys
 from pathlib import Path
-from typing import Optional
 
 import numpy as np
 
 from disdrodb.api.checks import (
     check_filepaths,
     check_start_end_time,
+    check_time,
     get_current_utc_time,
 )
-from disdrodb.api.info import get_start_end_time_from_filepaths
+from disdrodb.api.info import get_start_end_time_from_filepaths, group_filepaths
 from disdrodb.api.path import (
     define_campaign_dir,
     define_data_dir,
@@ -41,7 +40,7 @@ from disdrodb.api.path import (
 )
 from disdrodb.l0.l0_reader import define_readers_directory
 from disdrodb.utils.dict import extract_product_kwargs
-from disdrodb.utils.directories import list_files
+from disdrodb.utils.directories import list_files, remove_file_or_directories
 from disdrodb.utils.logger import (
     log_info,
 )
@@ -133,8 +132,8 @@ def find_files(
     station_name,
     product,
     debugging_mode: bool = False,
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
     glob_pattern=None,
     start_time=None,
     end_time=None,
@@ -292,6 +291,42 @@ def _open_raw_files(filepaths, data_source, campaign_name, station_name, metadat
     return ds
 
 
+def list_coordinates_names(ds):
+    """List coordinates of a xarray.Dataset not CF decoded !."""
+    coords = set()
+    for v in ds.variables:
+        attrs = ds[v].attrs
+        # auxiliary coordinates
+        if "coordinates" in attrs:
+            coords |= set(attrs["coordinates"].split())
+        # bounds variables
+        if "bounds" in attrs:
+            coords.add(attrs["bounds"])
+        # grid mapping
+        if "grid_mapping" in attrs:
+            coords.add(attrs["grid_mapping"])
+    return coords
+
+
+def subset_variables(ds, variables):
+    """Subset variables while keeping coordinates."""
+    # Ensure list
+    variables = list(variables)
+
+    # Always keep dimension variables
+    dim_vars = list(ds.dims)
+
+    # Variables referenced by CF relationships
+    coords = list_coordinates_names(ds)
+
+    # Union of everything we must keep
+    keep = set(variables) | set(dim_vars) | coords
+
+    # Only keep variables that exist
+    keep = [v for v in keep if v in list(ds.variables)]
+    return ds[keep]
+
+
 def filter_dataset_by_time(ds, start_time=None, end_time=None):
     """Subset an xarray.Dataset by time, robust to duplicated/non-monotonic indices.
 
@@ -321,6 +356,84 @@ def filter_dataset_by_time(ds, start_time=None, end_time=None):
     return ds.isel(time=np.where(mask)[0])
 
 
+def open_parquet_files(
+    filepaths,
+    variables=None,
+    start_time=None,
+    end_time=None,
+    time_col="time",
+    use_threads=True,
+):
+    """Open Parquet files."""
+    import pyarrow.dataset as ds
+
+    # Open dataset
+    dataset = ds.dataset(
+        filepaths,
+        format="parquet",
+    )
+
+    # Define filters
+    filters = []
+    if start_time is not None:
+        start_time = check_time(start_time)
+        filters.append(ds.field(time_col) >= start_time)
+
+    if end_time is not None:
+        end_time = check_time(end_time)
+        filters.append(ds.field(time_col) <= end_time)
+
+    # Combine filters if any exist
+    filter_expr = None
+    if filters:
+        filter_expr = filters[0]
+        for f in filters[1:]:
+            filter_expr = filter_expr & f
+
+    # Read table and convert to pandas
+    df = dataset.to_table(
+        columns=variables,
+        filter=filter_expr,
+        use_threads=use_threads,
+    ).to_pandas()
+    return df
+
+
+def ensure_safe_open_mfdataset(function):
+    """Decorator to ensure safe xarray open_mfdataset.
+
+    parallel argument is changed to False if:
+    - dask threading or single-threaded is active
+    - distributed multiprocessing with more than 1 thread per process
+
+    parallel argument is allowed to be True only if:
+    - distributed multiprocessing with only 1 thread per process
+    """
+    import dask
+
+    from disdrodb.utils.dask import check_parallel_validity
+
+    @functools.wraps(function)
+    def wrapper(*args, **kwargs):
+        # Check if it must be a delayed function
+        parallel = kwargs.get("parallel", False)
+        parallel = check_parallel_validity(parallel)
+        kwargs["parallel"] = parallel
+
+        # If parallel is True at this stage, means being using
+        # multiprocessing or dask.distributed with single thread
+        if parallel:
+            return function(*args, **kwargs)
+
+        # Call function with single threading
+        with dask.config.set(scheduler="single-threaded"):  # "synchronous"
+            result = function(*args, **kwargs)
+        return result
+
+    return wrapper
+
+
+@ensure_safe_open_mfdataset
 def open_netcdf_files(
     filepaths,
     chunks=-1,
@@ -329,46 +442,101 @@ def open_netcdf_files(
     variables=None,
     parallel=False,
     compute=True,
+    engine="netcdf4",
     **open_kwargs,
 ):
     """Open DISDRODB netCDF files using xarray.
 
-    Using combine="nested" and join="outer" ensure that duplicated timesteps are not overwritten!
+    Using data_vars="minimal", coords="minimal", compat="override"
+    --> will only concatenate those variables with the time dimension,
+    --> will skip any checking for variables that don't have a time dimension
+       (simply pick the variable from the first file).
+    https://github.com/pydata/xarray/issues/1385#issuecomment-1958761334
+
+    Using combine="nested" and join="outer" ensure that duplicated timesteps
+    are not overwritten!
+
+    When decode_cf=False
+    --> lat,lon are data_vars and get concatenated without any checking or reading
+    When decode_cf=True
+    --> lat, lon are promoted to coords, then get checked for equality across all files
+
+    For L0B product, if sample_interval variable is present and varies with time,
+    this function concatenate the variable over time without problems.
+    For L0C product, if sample_interval changes across listed files,
+    only sample_interval of first file is reported.
+    --> open_dataset take care of just providing filepaths of files with same sample interval.
+    In L1 and L2 processing, only filepaths of files with same sample interval
+    must be passed to this function.
+
     """
     import xarray as xr
 
     # Ensure variables is a list
-    if variables is not None and isinstance(variables, str):
-        variables = [variables]
+    if variables is not None:
+        if isinstance(variables, str):
+            variables = [variables]
+        variables = np.unique(variables).tolist()
+
     # Define preprocessing function for parallel opening
-    preprocess = (lambda ds: ds[variables]) if parallel and variables is not None else None
+    if parallel and variables is not None:
+
+        def preprocess(ds):
+            return subset_variables(ds, variables)
+
+    else:
+        preprocess = None
 
     # Open netcdf
+    xr.set_options(use_new_combine_kwarg_defaults=True)
     ds = xr.open_mfdataset(
         filepaths,
         chunks=chunks,
-        data_vars="all",
         combine="nested",
-        join="outer",
         concat_dim="time",
-        engine="netcdf4",
-        parallel=parallel,
-        preprocess=preprocess,
-        compat="no_conflicts",
+        data_vars="minimal",  # ["sample_interval"], "all" would concat all across time
+        coords="minimal",
+        join="outer",  # "exact"
+        compat="override",  # "no_conflicts" slows down
         combine_attrs="override",
-        coords="different",  # maybe minimal? would remove lon/lat/alt?
+        preprocess=preprocess,  # only if parallel=True
+        engine=engine,
+        parallel=parallel,
+        decode_cf=False,  # assume encoding do not vary across files (e.g. "time" units)
+        decode_coords=False,  # no effect if decode_cf=False
         decode_timedelta=False,
         cache=False,
         autoclose=True,
         **open_kwargs,
     )
-    # - Subset variables
+
+    # Decode CF
+    # - Set to coordinates the variables
+    #   - latitude/longitude/altitude
+    #   - sample_interval
+    #   - diameter/velocity bin width/upper/lower
+    ds = xr.decode_cf(ds, decode_times=True, decode_coords=True, decode_timedelta=False)
+
+    # Subset variables
+    # --> After decoding CF, when coordinates are properly set
+    # --> Othewerwise, coordinate variables would be removed unless listed in variables
     if variables is not None and preprocess is None:
+        variables = [var for var in variables if var in ds]
         ds = ds[variables]
-    # - Subset time
+
+    # Subset time
     if start_time is not None or end_time is not None:
         ds = filter_dataset_by_time(ds, start_time=start_time, end_time=end_time)
-    # - If compute=True, load in memory and close connections to files
+
+    # Ensure coordinates are already loaded in memory
+    for coord in list(ds.coords):
+        ds[coord] = ds[coord].load()
+
+    # Update time coverage attributes
+    ds.attrs["time_coverage_start"] = str(ds.disdrodb.start_time)
+    ds.attrs["time_coverage_end"] = str(ds.disdrodb.end_time)
+
+    # If compute=True, load in memory and close connections to files
     if compute:
         dataset = ds.compute()
         ds.close()
@@ -386,8 +554,8 @@ def open_dataset(
     product,
     product_kwargs=None,
     debugging_mode: bool = False,
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
     chunks=-1,
     parallel=False,
     compute=False,
@@ -429,7 +597,7 @@ def open_dataset(
     xarray.Dataset
 
     """
-    from disdrodb.l0.l0a_processing import read_l0a_dataframe
+    import xarray as xr
 
     # Extract product kwargs from open_kwargs
     product_kwargs = extract_product_kwargs(open_kwargs, product=product)
@@ -463,9 +631,46 @@ def open_dataset(
 
     # Open L0A Parquet files
     if product == "L0A":
-        return read_l0a_dataframe(filepaths)
+        df = open_parquet_files(
+            filepaths=filepaths,
+            variables=variables,
+            start_time=start_time,
+            end_time=end_time,
+            use_threads=parallel,
+        )
+        return df
 
     # Open DISDRODB netCDF files using xarray
+    # - Special handling for L0C product with possible multiple sample intervals
+    if product == "L0C":
+        dict_sample_intervals = group_filepaths(filepaths, groups="sample_interval")
+        if len(dict_sample_intervals) > 1:
+            # Open separately each sample interval
+            list_ds = [
+                open_netcdf_files(
+                    filepaths=filepaths,
+                    chunks=chunks,
+                    start_time=start_time,
+                    end_time=end_time,
+                    variables=variables,
+                    parallel=parallel,
+                    compute=compute,
+                    **open_kwargs,
+                )
+                for filepaths in dict_sample_intervals.values()
+            ]
+            # Expand sample_interval coordinate for each dataset
+            list_ds = [ds.assign_coords(sample_interval=ds.sample_interval.expand_dims(time=ds.time)) for ds in list_ds]
+            # Concatenate along time dimension and sort by time
+            ds = xr.concat(list_ds, dim="time")
+            ds.attrs["measurement_interval"] = list(dict_sample_intervals)
+            ds = ds.sortby("time")
+            # Update time coverage attributes
+            ds.attrs["time_coverage_start"] = str(ds.disdrodb.start_time)
+            ds.attrs["time_coverage_end"] = str(ds.disdrodb.end_time)
+            return ds
+
+    # Otherwise, open all files together
     ds = open_netcdf_files(
         filepaths=filepaths,
         chunks=chunks,
@@ -476,10 +681,6 @@ def open_dataset(
         compute=compute,
         **open_kwargs,
     )
-
-    # Ensure coordinates in memory
-    # for coord in list(ds.coords):
-    #     ds[coord] = ds[coord].compute()
     return ds
 
 
@@ -509,7 +710,7 @@ def remove_product(
         **product_kwargs,
     )
     log_info(logger=logger, msg="Removal of {product} files started.", verbose=verbose)
-    shutil.rmtree(data_dir)
+    remove_file_or_directories(data_dir)
     log_info(logger=logger, msg="Removal of {product} files ended.", verbose=verbose)
 
 

disdrodb/api/path.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,13 +15,13 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Define paths within the DISDRODB infrastructure."""
+
 import os
 
 from disdrodb.configs import get_data_archive_dir, get_metadata_archive_dir
 from disdrodb.constants import ARCHIVE_VERSION
 from disdrodb.utils.directories import check_directory_exists
 from disdrodb.utils.time import (
-    ensure_sample_interval_in_seconds,
     get_file_start_end_time,
     seconds_to_temporal_resolution,
 )
@@ -805,10 +803,8 @@ def define_l0b_filename(ds, campaign_name: str, station_name: str) -> str:
     return filename
 
 
-def define_l0c_filename(ds, campaign_name: str, station_name: str) -> str:
+def define_l0c_filename(ds, campaign_name: str, station_name: str, sample_interval: str) -> str:
     """Define L0C file name."""
-    # TODO: add sample_interval as function argument
-    sample_interval = int(ensure_sample_interval_in_seconds(ds["sample_interval"]).data.item())
     temporal_resolution = define_temporal_resolution(sample_interval, rolling=False)
     starting_time, ending_time = get_file_start_end_time(ds)
     starting_time = starting_time.strftime("%Y%m%d%H%M%S")

disdrodb/cli/disdrodb_check_metadata_archive.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Wrapper to check DISDRODB Metadata Archive Compliance from terminal."""
+
 import sys
 
 import click
@@ -34,7 +35,7 @@ sys.tracebacklimit = 0  # avoid full traceback error if occur
     help="Whether to raise error of finish the check",
 )
 def disdrodb_check_metadata_archive(metadata_archive_dir=None, raise_error=True):
-    """Run the DISDRODB Metadata Archive Checks."""
+    """Validate the DISDRODB Metadata Archive."""
     from disdrodb.metadata.checks import check_metadata_archive
 
     metadata_archive_dir = parse_archive_dir(metadata_archive_dir)