disdrodb 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

disdrodb/__init__.py

@@ -20,6 +20,8 @@ import contextlib
 import os
 from importlib.metadata import PackageNotFoundError, version
 
+os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
+
 import disdrodb.accessor  # noqa
 from disdrodb._config import config  # noqa
 from disdrodb.api.configs import available_sensor_names

disdrodb/_config.py

@@ -18,6 +18,7 @@
 
 See https://donfig.readthedocs.io/en/latest/configuration.html for more info.
 """
+
 from donfig import Config
 
 from disdrodb.configs import read_configs

disdrodb/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.3.0'
-__version_tuple__ = version_tuple = (0, 3, 0)
+__version__ = version = '0.4.0'
+__version_tuple__ = version_tuple = (0, 4, 0)
 
 __commit_id__ = commit_id = None

disdrodb/accessor/__init__.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """This directory defines DISDRODB xarray accessors."""
+
 from .methods import DISDRODB_DataArray_Accessor, DISDRODB_Dataset_Accessor
 
 __all__ = ["DISDRODB_DataArray_Accessor", "DISDRODB_Dataset_Accessor"]

disdrodb/accessor/methods.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """This module defines DISDRODB xarray accessors."""
+
 import xarray as xr
 
 

disdrodb/api/checks.py

@@ -15,12 +15,12 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """DISDRODB Checks Functions."""
+
 import datetime
 import difflib
 import logging
 import os
 import re
-import sys
 import warnings
 
 import numpy as np
@@ -565,9 +565,7 @@ def check_filepaths(filepaths):
 
 def get_current_utc_time():
     """Get current UTC time."""
-    if sys.version_info >= (3, 11):
-        return datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
-    return datetime.datetime.utcnow()
+    return datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
 
 
 def check_start_end_time(start_time, end_time):

disdrodb/api/io.py

@@ -15,21 +15,23 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Routines to list and open DISDRODB products."""
+
 import datetime
+import functools
 import os
 import subprocess
 import sys
 from pathlib import Path
-from typing import Optional
 
 import numpy as np
 
 from disdrodb.api.checks import (
     check_filepaths,
     check_start_end_time,
+    check_time,
     get_current_utc_time,
 )
-from disdrodb.api.info import get_start_end_time_from_filepaths
+from disdrodb.api.info import get_start_end_time_from_filepaths, group_filepaths
 from disdrodb.api.path import (
     define_campaign_dir,
     define_data_dir,
@@ -130,8 +132,8 @@ def find_files(
     station_name,
     product,
     debugging_mode: bool = False,
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
     glob_pattern=None,
     start_time=None,
     end_time=None,
@@ -289,6 +291,42 @@ def _open_raw_files(filepaths, data_source, campaign_name, station_name, metadat
     return ds
 
 
+def list_coordinates_names(ds):
+    """List coordinates of a xarray.Dataset not CF decoded !."""
+    coords = set()
+    for v in ds.variables:
+        attrs = ds[v].attrs
+        # auxiliary coordinates
+        if "coordinates" in attrs:
+            coords |= set(attrs["coordinates"].split())
+        # bounds variables
+        if "bounds" in attrs:
+            coords.add(attrs["bounds"])
+        # grid mapping
+        if "grid_mapping" in attrs:
+            coords.add(attrs["grid_mapping"])
+    return coords
+
+
+def subset_variables(ds, variables):
+    """Subset variables while keeping coordinates."""
+    # Ensure list
+    variables = list(variables)
+
+    # Always keep dimension variables
+    dim_vars = list(ds.dims)
+
+    # Variables referenced by CF relationships
+    coords = list_coordinates_names(ds)
+
+    # Union of everything we must keep
+    keep = set(variables) | set(dim_vars) | coords
+
+    # Only keep variables that exist
+    keep = [v for v in keep if v in list(ds.variables)]
+    return ds[keep]
+
+
 def filter_dataset_by_time(ds, start_time=None, end_time=None):
     """Subset an xarray.Dataset by time, robust to duplicated/non-monotonic indices.
 
@@ -318,6 +356,84 @@ def filter_dataset_by_time(ds, start_time=None, end_time=None):
     return ds.isel(time=np.where(mask)[0])
 
 
+def open_parquet_files(
+    filepaths,
+    variables=None,
+    start_time=None,
+    end_time=None,
+    time_col="time",
+    use_threads=True,
+):
+    """Open Parquet files."""
+    import pyarrow.dataset as ds
+
+    # Open dataset
+    dataset = ds.dataset(
+        filepaths,
+        format="parquet",
+    )
+
+    # Define filters
+    filters = []
+    if start_time is not None:
+        start_time = check_time(start_time)
+        filters.append(ds.field(time_col) >= start_time)
+
+    if end_time is not None:
+        end_time = check_time(end_time)
+        filters.append(ds.field(time_col) <= end_time)
+
+    # Combine filters if any exist
+    filter_expr = None
+    if filters:
+        filter_expr = filters[0]
+        for f in filters[1:]:
+            filter_expr = filter_expr & f
+
+    # Read table and convert to pandas
+    df = dataset.to_table(
+        columns=variables,
+        filter=filter_expr,
+        use_threads=use_threads,
+    ).to_pandas()
+    return df
+
+
+def ensure_safe_open_mfdataset(function):
+    """Decorator to ensure safe xarray open_mfdataset.
+
+    parallel argument is changed to False if:
+    - dask threading or single-threaded is active
+    - distributed multiprocessing with more than 1 thread per process
+
+    parallel argument is allowed to be True only if:
+    - distributed multiprocessing with only 1 thread per process
+    """
+    import dask
+
+    from disdrodb.utils.dask import check_parallel_validity
+
+    @functools.wraps(function)
+    def wrapper(*args, **kwargs):
+        # Check if it must be a delayed function
+        parallel = kwargs.get("parallel", False)
+        parallel = check_parallel_validity(parallel)
+        kwargs["parallel"] = parallel
+
+        # If parallel is True at this stage, means being using
+        # multiprocessing or dask.distributed with single thread
+        if parallel:
+            return function(*args, **kwargs)
+
+        # Call function with single threading
+        with dask.config.set(scheduler="single-threaded"):  # "synchronous"
+            result = function(*args, **kwargs)
+        return result
+
+    return wrapper
+
+
+@ensure_safe_open_mfdataset
 def open_netcdf_files(
     filepaths,
     chunks=-1,
@@ -326,11 +442,33 @@ def open_netcdf_files(
     variables=None,
     parallel=False,
     compute=True,
+    engine="netcdf4",
     **open_kwargs,
 ):
     """Open DISDRODB netCDF files using xarray.
 
-    Using combine="nested" and join="outer" ensure that duplicated timesteps are not overwritten!
+    Using data_vars="minimal", coords="minimal", compat="override"
+    --> will only concatenate those variables with the time dimension,
+    --> will skip any checking for variables that don't have a time dimension
+       (simply pick the variable from the first file).
+    https://github.com/pydata/xarray/issues/1385#issuecomment-1958761334
+
+    Using combine="nested" and join="outer" ensure that duplicated timesteps
+    are not overwritten!
+
+    When decode_cf=False
+    --> lat,lon are data_vars and get concatenated without any checking or reading
+    When decode_cf=True
+    --> lat, lon are promoted to coords, then get checked for equality across all files
+
+    For L0B product, if sample_interval variable is present and varies with time,
+    this function concatenate the variable over time without problems.
+    For L0C product, if sample_interval changes across listed files,
+    only sample_interval of first file is reported.
+    --> open_dataset take care of just providing filepaths of files with same sample interval.
+    In L1 and L2 processing, only filepaths of files with same sample interval
+    must be passed to this function.
+
     """
     import xarray as xr
 
@@ -341,35 +479,64 @@ def open_netcdf_files(
         variables = np.unique(variables).tolist()
 
     # Define preprocessing function for parallel opening
-    preprocess = (lambda ds: ds[variables]) if parallel and variables is not None else None
+    if parallel and variables is not None:
+
+        def preprocess(ds):
+            return subset_variables(ds, variables)
+
+    else:
+        preprocess = None
 
     # Open netcdf
+    xr.set_options(use_new_combine_kwarg_defaults=True)
     ds = xr.open_mfdataset(
         filepaths,
         chunks=chunks,
-        data_vars="all",
         combine="nested",
-        join="outer",
         concat_dim="time",
-        engine="netcdf4",
-        parallel=parallel,
-        preprocess=preprocess,
-        compat="no_conflicts",
+        data_vars="minimal",  # ["sample_interval"], "all" would concat all across time
+        coords="minimal",
+        join="outer",  # "exact"
+        compat="override",  # "no_conflicts" slows down
         combine_attrs="override",
-        coords="different",  # maybe minimal? would remove lon/lat/alt?
+        preprocess=preprocess,  # only if parallel=True
+        engine=engine,
+        parallel=parallel,
+        decode_cf=False,  # assume encoding do not vary across files (e.g. "time" units)
+        decode_coords=False,  # no effect if decode_cf=False
         decode_timedelta=False,
         cache=False,
         autoclose=True,
         **open_kwargs,
     )
-    # - Subset variables
+
+    # Decode CF
+    # - Set to coordinates the variables
+    #   - latitude/longitude/altitude
+    #   - sample_interval
+    #   - diameter/velocity bin width/upper/lower
+    ds = xr.decode_cf(ds, decode_times=True, decode_coords=True, decode_timedelta=False)
+
+    # Subset variables
+    # --> After decoding CF, when coordinates are properly set
+    # --> Othewerwise, coordinate variables would be removed unless listed in variables
     if variables is not None and preprocess is None:
         variables = [var for var in variables if var in ds]
         ds = ds[variables]
-    # - Subset time
+
+    # Subset time
     if start_time is not None or end_time is not None:
         ds = filter_dataset_by_time(ds, start_time=start_time, end_time=end_time)
-    # - If compute=True, load in memory and close connections to files
+
+    # Ensure coordinates are already loaded in memory
+    for coord in list(ds.coords):
+        ds[coord] = ds[coord].load()
+
+    # Update time coverage attributes
+    ds.attrs["time_coverage_start"] = str(ds.disdrodb.start_time)
+    ds.attrs["time_coverage_end"] = str(ds.disdrodb.end_time)
+
+    # If compute=True, load in memory and close connections to files
     if compute:
         dataset = ds.compute()
         ds.close()
@@ -387,8 +554,8 @@ def open_dataset(
     product,
     product_kwargs=None,
     debugging_mode: bool = False,
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
     chunks=-1,
     parallel=False,
     compute=False,
@@ -430,7 +597,7 @@ def open_dataset(
     xarray.Dataset
 
     """
-    from disdrodb.l0.l0a_processing import read_l0a_dataframe
+    import xarray as xr
 
     # Extract product kwargs from open_kwargs
     product_kwargs = extract_product_kwargs(open_kwargs, product=product)
@@ -464,9 +631,46 @@ def open_dataset(
 
     # Open L0A Parquet files
     if product == "L0A":
-        return read_l0a_dataframe(filepaths)
+        df = open_parquet_files(
+            filepaths=filepaths,
+            variables=variables,
+            start_time=start_time,
+            end_time=end_time,
+            use_threads=parallel,
+        )
+        return df
 
     # Open DISDRODB netCDF files using xarray
+    # - Special handling for L0C product with possible multiple sample intervals
+    if product == "L0C":
+        dict_sample_intervals = group_filepaths(filepaths, groups="sample_interval")
+        if len(dict_sample_intervals) > 1:
+            # Open separately each sample interval
+            list_ds = [
+                open_netcdf_files(
+                    filepaths=filepaths,
+                    chunks=chunks,
+                    start_time=start_time,
+                    end_time=end_time,
+                    variables=variables,
+                    parallel=parallel,
+                    compute=compute,
+                    **open_kwargs,
+                )
+                for filepaths in dict_sample_intervals.values()
+            ]
+            # Expand sample_interval coordinate for each dataset
+            list_ds = [ds.assign_coords(sample_interval=ds.sample_interval.expand_dims(time=ds.time)) for ds in list_ds]
+            # Concatenate along time dimension and sort by time
+            ds = xr.concat(list_ds, dim="time")
+            ds.attrs["measurement_interval"] = list(dict_sample_intervals)
+            ds = ds.sortby("time")
+            # Update time coverage attributes
+            ds.attrs["time_coverage_start"] = str(ds.disdrodb.start_time)
+            ds.attrs["time_coverage_end"] = str(ds.disdrodb.end_time)
+            return ds
+
+    # Otherwise, open all files together
     ds = open_netcdf_files(
         filepaths=filepaths,
         chunks=chunks,
@@ -477,10 +681,6 @@ def open_dataset(
         compute=compute,
         **open_kwargs,
     )
-
-    # Ensure coordinates in memory
-    # for coord in list(ds.coords):
-    #     ds[coord] = ds[coord].compute()
     return ds
 
 

disdrodb/api/path.py

@@ -15,13 +15,13 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Define paths within the DISDRODB infrastructure."""
+
 import os
 
 from disdrodb.configs import get_data_archive_dir, get_metadata_archive_dir
 from disdrodb.constants import ARCHIVE_VERSION
 from disdrodb.utils.directories import check_directory_exists
 from disdrodb.utils.time import (
-    ensure_sample_interval_in_seconds,
     get_file_start_end_time,
     seconds_to_temporal_resolution,
 )
@@ -803,10 +803,8 @@ def define_l0b_filename(ds, campaign_name: str, station_name: str) -> str:
     return filename
 
 
-def define_l0c_filename(ds, campaign_name: str, station_name: str) -> str:
+def define_l0c_filename(ds, campaign_name: str, station_name: str, sample_interval: str) -> str:
     """Define L0C file name."""
-    # TODO: add sample_interval as function argument
-    sample_interval = int(ensure_sample_interval_in_seconds(ds["sample_interval"]).data.item())
     temporal_resolution = define_temporal_resolution(sample_interval, rolling=False)
     starting_time, ending_time = get_file_start_end_time(ds)
     starting_time = starting_time.strftime("%Y%m%d%H%M%S")

disdrodb/cli/disdrodb_check_metadata_archive.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Wrapper to check DISDRODB Metadata Archive Compliance from terminal."""
+
 import sys
 
 import click

disdrodb/cli/disdrodb_check_products_options.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Script to check the validity of the DISDRODB products configuration files."""
+
 import sys
 
 import click

disdrodb/cli/disdrodb_create_summary.py

@@ -15,8 +15,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Script to create summary figures and tables for a DISDRODB stationn."""
+
 import sys
-from typing import Optional
 
 import click
 
@@ -49,15 +49,15 @@ sys.tracebacklimit = 0  # avoid full traceback error if occur
 )
 def disdrodb_create_summary(
     # Stations options
-    data_sources: Optional[str] = None,
-    campaign_names: Optional[str] = None,
-    station_names: Optional[str] = None,
+    data_sources: str | None = None,
+    campaign_names: str | None = None,
+    station_names: str | None = None,
     # Processing options:
     parallel=False,
     temporal_resolution="1MIN",
     # DISDRODB root directories
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
 ):
     """Create summary figures and tables for DISDRODB stations.
 

disdrodb/cli/disdrodb_create_summary_station.py

@@ -15,8 +15,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Script to create summary figures and tables for a DISDRODB station."""
+
 import sys
-from typing import Optional
 
 import click
 
@@ -53,7 +53,7 @@ def disdrodb_create_summary_station(
     parallel=False,
     temporal_resolution="1MIN",
     # DISDRODB root directories
-    data_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
 ):
     """Create summary figures and tables for a specific DISDRODB station.
 

disdrodb/cli/disdrodb_data_archive_directory.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Routine to print the DISDRODB Data Archive directory."""
+
 import sys
 
 import click

disdrodb/cli/disdrodb_download_archive.py

@@ -17,7 +17,6 @@
 """Wrapper to download stations from the DISDRODB Decentralized Data Archive."""
 
 import sys
-from typing import Optional
 
 import click
 
@@ -38,11 +37,11 @@ sys.tracebacklimit = 0  # avoid full traceback error if occur
 @click_metadata_archive_dir_option
 @click_download_options
 def disdrodb_download_archive(
-    data_sources: Optional[str] = None,
-    campaign_names: Optional[str] = None,
-    station_names: Optional[str] = None,
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
+    data_sources: str | None = None,
+    campaign_names: str | None = None,
+    station_names: str | None = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
     force: bool = False,
 ):
     """Download raw data for multiple DISDRODB stations from the DISDRODB Decentralized Data Archive.

disdrodb/cli/disdrodb_download_metadata_archive.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Routine to download the DISDRODB Metadata Data Archive."""
+
 import sys
 from pathlib import Path
 

disdrodb/cli/disdrodb_download_station.py

@@ -17,7 +17,6 @@
 """Routine to download station data from the DISDRODB Decentralized Data Archive."""
 
 import sys
-from typing import Optional
 
 import click
 
@@ -41,8 +40,8 @@ def disdrodb_download_station(
     data_source: str,
     campaign_name: str,
     station_name: str,
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
     force: bool = False,
 ):
     """Download raw data of a single station from the DISDRODB Decentralized Data Archive.

disdrodb/cli/disdrodb_initialize_station.py

@@ -15,8 +15,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Script to initialize the DISDRODB station directory structure."""
+
 import sys
-from typing import Optional
 
 import click
 
@@ -43,8 +43,8 @@ def disdrodb_initialize_station(
     campaign_name: str,
     station_name: str,
     # DISDRODB root directories
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
 ):
     """Initialize the DISDRODB directory structure for a new station.
 

disdrodb/cli/disdrodb_metadata_archive_directory.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Routine to print the DISDRODB Data Archive directory."""
+
 import sys
 
 import click

disdrodb/cli/disdrodb_open_data_archive.py

@@ -17,7 +17,6 @@
 """Routine to open the DISDRODB Data Archive."""
 
 import sys
-from typing import Optional
 
 import click
 
@@ -32,7 +31,7 @@ sys.tracebacklimit = 0  # avoid full traceback error if occur
 @click.command()
 @click_data_archive_dir_option
 def disdrodb_open_data_archive(
-    data_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
 ):
     """Open the DISDRODB Data Archive directory in the system file explorer.
 

disdrodb/cli/disdrodb_open_logs_directory.py

@@ -17,7 +17,6 @@
 """Routine to open the DISDRODB Data Archive logs directory."""
 
 import sys
-from typing import Optional
 
 import click
 
@@ -36,8 +35,8 @@ sys.tracebacklimit = 0  # avoid full traceback error if occur
 def disdrodb_open_logs_directory(
     data_source: str,
     campaign_name: str,
-    station_name: Optional[str] = None,
-    data_archive_dir: Optional[str] = None,
+    station_name: str | None = None,
+    data_archive_dir: str | None = None,
 ):
     """Open the DISDRODB Data Archive station logs directory in the system file explorer.
 

disdrodb/cli/disdrodb_open_metadata_archive.py

@@ -17,7 +17,6 @@
 """Routine to open the DISDRODB Metadata Data Archive."""
 
 import sys
-from typing import Optional
 
 import click
 
@@ -32,7 +31,7 @@ sys.tracebacklimit = 0  # avoid full traceback error if occur
 @click.command()
 @click_metadata_archive_dir_option
 def disdrodb_open_metadata_archive(
-    metadata_archive_dir: Optional[str] = None,
+    metadata_archive_dir: str | None = None,
 ):
     """Open the DISDRODB Metadata Archive directory in the system file explorer.