disdrodb 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

disdrodb/l0/readers/RD80/NOAA/PSL_RD80.py

@@ -18,12 +18,19 @@
 # -----------------------------------------------------------------------------.
 """DISDRODB Reader for NOAA PSL RD80 stations."""
 import os
+import re
 
+# Convert ParserWarning into an error
+import warnings
+
+import numpy as np
 import pandas as pd
 
 from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
 from disdrodb.l0.l0a_processing import read_raw_text_file
 
+warnings.simplefilter("error", pd.errors.ParserWarning)
+
 
 def read_new_format(filepath, logger):
     """Read new format."""
@@ -101,6 +108,14 @@ def read_new_format(filepath, logger):
 
     ##------------------------------------------------------------------------.
     #### Adapt the dataframe to adhere to DISDRODB L0 standards
+    # Retrieve date and hour information
+    with open(filepath) as f:
+        date_header = f.readline().strip()
+    match = re.search(r":\s*(\d+)\s*UTC", date_header)
+    if match:
+        date_hour_str = match.group(1)
+    else:
+        raise ValueError("Date information not found.")
 
     # Replace -99.9900 values with NaN
     columns_to_replace = ["Dmax", "RI", "RA", "Wg", "Z", "EF", "N0", "slope"]
@@ -116,28 +131,30 @@ def read_new_format(filepath, logger):
 
     # - Convert start/end MM:SS:SSS to timedelta
     def parse_time(t):
-        minutes, seconds, milliseconds = map(int, t.split(":"))
-        return pd.Timedelta(minutes=minutes, seconds=seconds, milliseconds=milliseconds)
+        try:
+            minutes, seconds, milliseconds = map(int, t.split(":"))
+            timedelta = pd.Timedelta(minutes=minutes, seconds=seconds, milliseconds=milliseconds)
+        except Exception:
+            timedelta = pd.Timedelta("NaT")
+        return timedelta
 
     df_time["start"] = df_time["start"].apply(parse_time)
     df_time["end"] = df_time["end"].apply(parse_time)
-    # - Wrap end time if it's less than start time (i.e., crosses 60:00 boundary)
-    # --> 00:00 --> 60:00
-    df_time.loc[df_time["end"] < df_time["start"], "end"] += pd.Timedelta(minutes=60)
-
-    # Compute sample_interval in seconds as integer
-    df["sample_interval"] = (df_time["end"] - df_time["start"]).dt.total_seconds().astype(int)
 
     # Define time
-    # - Extract date-hour
-    filename = os.path.basename(filepath)
-    if filename.startswith("lab") or filename.startswith("bao0") or filename.startswith("mdt0"):
-        date_hour_str = filename[4:11]
-    else:
-        date_hour_str = filename[3:10]
     date_hour = pd.to_datetime(date_hour_str, format="%y%j%H")
     df["time"] = date_hour + df_time["start"]
 
+    # Drop invalid timesteps
+    df_time = df_time[~np.isnan(df["time"])]
+    df = df[~np.isnan(df["time"])]
+
+    # Compute sample_interval in seconds as integer
+    # - Wrap end time if it's less than start time (i.e., crosses 60:00 boundary)
+    # --> 00:00 --> 60:00
+    df_time.loc[df_time["end"] < df_time["start"], "end"] += pd.Timedelta(minutes=60)
+    df["sample_interval"] = (df_time["end"] - df_time["start"]).dt.total_seconds().astype(int)
+
     # Create raw_drop_number column
     bin_columns = ["n" + str(i) for i in range(1, 21)]
     df_arr = df[bin_columns]

disdrodb/l0/routines.py

@@ -21,14 +21,13 @@
 import datetime
 import logging
 import os
+import shutil
 import time
 from typing import Optional
 
 import dask
 
-from disdrodb.api.checks import check_sensor_name
-
-# Directory
+from disdrodb.api.checks import check_sensor_name, check_station_inputs
 from disdrodb.api.create_directories import (
     create_l0_directory_structure,
     create_logs_directory,
@@ -41,6 +40,7 @@ from disdrodb.api.path import (
     define_l0b_filename,
     define_l0c_filename,
     define_metadata_filepath,
+    define_partitioning_tree,
 )
 from disdrodb.api.search import get_required_product
 from disdrodb.configs import get_data_archive_dir, get_folder_partitioning, get_metadata_archive_dir
@@ -53,7 +53,7 @@ from disdrodb.l0.l0a_processing import (
 )
 from disdrodb.l0.l0b_nc_processing import sanitize_ds
 from disdrodb.l0.l0b_processing import (
-    create_l0b_from_l0a,
+    generate_l0b,
     set_l0b_encodings,
     write_l0b,
 )
@@ -63,6 +63,7 @@ from disdrodb.l0.l0c_processing import (
     retrieve_possible_measurement_intervals,
 )
 from disdrodb.metadata import read_station_metadata
+from disdrodb.utils.attrs import set_disdrodb_attrs
 from disdrodb.utils.decorators import delayed_if_parallel, single_threaded_if_parallel
 
 # Logger
@@ -73,8 +74,6 @@ from disdrodb.utils.logger import (
     log_error,
     log_info,
 )
-
-# log_warning,
 from disdrodb.utils.writer import write_product
 from disdrodb.utils.yaml import read_yaml
 
@@ -124,7 +123,7 @@ def _generate_l0a(
     # Log start processing
     msg = f"{product} processing of {filename} has started."
     log_info(logger=logger, msg=msg, verbose=verbose)
-
+    success_flag = False
     ##------------------------------------------------------------------------.
     ### - Read raw file into a dataframe and sanitize for L0A format
     try:
@@ -144,6 +143,12 @@ def _generate_l0a(
         filepath = os.path.join(folder_path, filename)
         write_l0a(df=df, filepath=filepath, force=force, logger=logger, verbose=verbose)
 
+        ##--------------------------------------------------------------------.
+        #### - Define logger file final directory
+        if folder_partitioning != "":
+            log_dst_dir = define_file_folder_path(df, data_dir=logs_dir, folder_partitioning=folder_partitioning)
+            os.makedirs(log_dst_dir, exist_ok=True)
+
         ##--------------------------------------------------------------------.
         # Clean environment
         del df
@@ -151,6 +156,7 @@ def _generate_l0a(
         # Log end processing
         msg = f"{product} processing of {filename} has ended."
         log_info(logger=logger, msg=msg, verbose=verbose)
+        success_flag = True
 
     # Otherwise log the error
     except Exception as e:
@@ -161,6 +167,13 @@ def _generate_l0a(
     # Close the file logger
     close_logger(logger)
 
+    # Move logger file to correct partitioning directory
+    if success_flag and folder_partitioning != "" and logger_filepath is not None:
+        # Move logger file to correct partitioning directory
+        dst_filepath = os.path.join(log_dst_dir, os.path.basename(logger_filepath))
+        shutil.move(logger_filepath, dst_filepath)
+        logger_filepath = dst_filepath
+
     # Return the logger file path
     return logger_filepath
 
@@ -200,6 +213,7 @@ def _generate_l0b(
     # Log start processing
     msg = f"{product} processing of {filename} has started."
     log_info(logger=logger, msg=msg, verbose=verbose)
+    success_flag = False
 
     ##------------------------------------------------------------------------.
     # Retrieve sensor name
@@ -209,11 +223,11 @@ def _generate_l0b(
     ##------------------------------------------------------------------------.
     try:
         # Read L0A Apache Parquet file
-        df = read_l0a_dataframe(filepath, logger=logger, verbose=verbose, debugging_mode=debugging_mode)
+        df = read_l0a_dataframe(filepath, debugging_mode=debugging_mode)
 
         # -----------------------------------------------------------------.
         # Create xarray Dataset
-        ds = create_l0b_from_l0a(df=df, metadata=metadata, logger=logger, verbose=verbose)
+        ds = generate_l0b(df=df, metadata=metadata, logger=logger, verbose=verbose)
 
         # -----------------------------------------------------------------.
         # Write L0B netCDF4 dataset
@@ -222,6 +236,12 @@ def _generate_l0b(
         filepath = os.path.join(folder_path, filename)
         write_l0b(ds, filepath=filepath, force=force)
 
+        ##--------------------------------------------------------------------.
+        #### - Define logger file final directory
+        if folder_partitioning != "":
+            log_dst_dir = define_file_folder_path(ds, data_dir=logs_dir, folder_partitioning=folder_partitioning)
+            os.makedirs(log_dst_dir, exist_ok=True)
+
         ##--------------------------------------------------------------------.
         # Clean environment
         del ds, df
@@ -229,6 +249,7 @@ def _generate_l0b(
         # Log end processing
         msg = f"{product} processing of {filename} has ended."
         log_info(logger=logger, msg=msg, verbose=verbose)
+        success_flag = True
 
     # Otherwise log the error
     except Exception as e:
@@ -239,10 +260,19 @@ def _generate_l0b(
     # Close the file logger
     close_logger(logger)
 
+    # Move logger file to correct partitioning directory
+    if success_flag and folder_partitioning != "" and logger_filepath is not None:
+        # Move logger file to correct partitioning directory
+        dst_filepath = os.path.join(log_dst_dir, os.path.basename(logger_filepath))
+        shutil.move(logger_filepath, dst_filepath)
+        logger_filepath = dst_filepath
+
     # Return the logger file path
     return logger_filepath
 
 
+@delayed_if_parallel
+@single_threaded_if_parallel
 def _generate_l0b_from_nc(
     filepath,
     data_dir,
@@ -282,6 +312,7 @@ def _generate_l0b_from_nc(
     # Log start processing
     msg = f"{product} processing of {filename} has started."
     log_info(logger=logger, msg=msg, verbose=verbose)
+    success_flag = False
 
     ##------------------------------------------------------------------------.
     ### - Read raw netCDF and sanitize for L0B format
@@ -303,6 +334,12 @@ def _generate_l0b_from_nc(
         filepath = os.path.join(folder_path, filename)
         write_l0b(ds, filepath=filepath, force=force)
 
+        ##--------------------------------------------------------------------.
+        #### - Define logger file final directory
+        if folder_partitioning != "":
+            log_dst_dir = define_file_folder_path(ds, data_dir=logs_dir, folder_partitioning=folder_partitioning)
+            os.makedirs(log_dst_dir, exist_ok=True)
+
         ##--------------------------------------------------------------------.
         # Clean environment
         del ds
@@ -310,6 +347,7 @@ def _generate_l0b_from_nc(
         # Log end processing
         msg = f"L0B processing of {filename} has ended."
         log_info(logger=logger, msg=msg, verbose=verbose)
+        success_flag = True
 
     # Otherwise log the error
     except Exception as e:
@@ -320,6 +358,13 @@ def _generate_l0b_from_nc(
     # Close the file logger
     close_logger(logger)
 
+    # Move logger file to correct partitioning directory
+    if success_flag and folder_partitioning != "" and logger_filepath is not None:
+        # Move logger file to correct partitioning directory
+        dst_filepath = os.path.join(log_dst_dir, os.path.basename(logger_filepath))
+        shutil.move(logger_filepath, dst_filepath)
+        logger_filepath = dst_filepath
+
     # Return the logger file path
     return logger_filepath
 
@@ -358,6 +403,7 @@ def _generate_l0c(
     # Log start processing
     msg = f"{product} processing for {day} has started."
     log_info(logger=logger, msg=msg, verbose=verbose)
+    success_flag = False
 
     ##------------------------------------------------------------------------.
     ### Core computation
@@ -388,21 +434,35 @@ def _generate_l0c(
 
                 # Set encodings
                 ds = set_l0b_encodings(ds=ds, sensor_name=sensor_name)
+                # Update global attributes
+                ds = set_disdrodb_attrs(ds, product=product)
 
-                # Define filepath
+                # Define product filepath
                 filename = define_l0c_filename(ds, campaign_name=campaign_name, station_name=station_name)
                 folder_path = define_file_folder_path(ds, data_dir=data_dir, folder_partitioning=folder_partitioning)
                 filepath = os.path.join(folder_path, filename)
 
                 # Write to disk
-                write_product(ds, product=product, filepath=filepath, force=force)
+                write_product(ds, filepath=filepath, force=force)
 
         # Clean environment
         del ds
 
+        ##--------------------------------------------------------------------.
+        #### - Define logger file final directory
+        if folder_partitioning != "":
+            print(day)
+            dirtree = define_partitioning_tree(
+                time=datetime.datetime.strptime("2022-03-22", "%Y-%m-%d"),
+                folder_partitioning=folder_partitioning,
+            )
+            log_dst_dir = os.path.join(logs_dir, dirtree)
+            os.makedirs(log_dst_dir, exist_ok=True)
+
         # Log end processing
         msg = f"{product} processing for {day} has ended."
         log_info(logger=logger, msg=msg, verbose=verbose)
+        success_flag = True
 
     ##--------------------------------------------------------------------.
     # Otherwise log the error
@@ -414,6 +474,13 @@ def _generate_l0c(
     # Close the file logger
     close_logger(logger)
 
+    # Move logger file to correct partitioning directory
+    if success_flag and folder_partitioning != "" and logger_filepath is not None:
+        # Move logger file to correct partitioning directory
+        dst_filepath = os.path.join(log_dst_dir, os.path.basename(logger_filepath))
+        shutil.move(logger_filepath, dst_filepath)
+        logger_filepath = dst_filepath
+
     # Return the logger file path
     return logger_filepath
 
@@ -474,6 +541,15 @@ def run_l0a_station(
     data_archive_dir = get_data_archive_dir(data_archive_dir)
     metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
 
+    # Check valid data_source, campaign_name, and station_name
+    check_station_inputs(
+        metadata_archive_dir=metadata_archive_dir,
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+    )
+
+    # ------------------------------------------------------------------------.
     # Read metadata
     metadata = read_station_metadata(
         metadata_archive_dir=metadata_archive_dir,
@@ -652,7 +728,7 @@ def run_l0b_station(
         and multi-threading will be automatically exploited to speed up I/O tasks.
     debugging_mode : bool, optional
         If ``True``, the amount of data processed will be reduced.
-        Only the first 100 rows of 3 L0A files will be processed. The default value is ``False``.
+        Only 100 rows sampled from 3 L0A files will be processed. The default value is ``False``.
     remove_l0a: bool, optional
         Whether to remove the processed L0A files. The default value is ``False``.
     data_archive_dir : str, optional
@@ -669,6 +745,13 @@ def run_l0b_station(
     # Retrieve DISDRODB Metadata Archive directory
     metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
 
+    # Check valid data_source, campaign_name, and station_name
+    check_station_inputs(
+        metadata_archive_dir=metadata_archive_dir,
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+    )
     # -----------------------------------------------------------------.
     # Retrieve metadata
     metadata = read_station_metadata(
@@ -731,7 +814,7 @@ def run_l0b_station(
     # If no data available, print error message and return None
     if flag_not_available_data:
         msg = (
-            f"{product} processing of {data_source} {campaign_name} {station_name}"
+            f"{product} processing of {data_source} {campaign_name} {station_name} "
             + f"has not been launched because of missing {required_product} data."
         )
         print(msg)
@@ -899,6 +982,14 @@ def run_l0c_station(
     # Retrieve DISDRODB Metadata Archive directory
     metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
 
+    # Check valid data_source, campaign_name, and station_name
+    check_station_inputs(
+        metadata_archive_dir=metadata_archive_dir,
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+    )
+
     # ------------------------------------------------------------------------.
     # Start processing
     t_i = time.time()
@@ -957,7 +1048,7 @@ def run_l0c_station(
     # If no data available, print error message and return None
     if flag_not_available_data:
         msg = (
-            f"{product} processing of {data_source} {campaign_name} {station_name}"
+            f"{product} processing of {data_source} {campaign_name} {station_name} "
             + f"has not been launched because of missing {required_product} data."
         )
         print(msg)

disdrodb/l1/__init__.py

@@ -15,3 +15,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """DISDRODB L1 module."""
+from disdrodb.l1.processing import generate_l1
+
+__all__ = [
+    "generate_l1",
+]

disdrodb/l1/filters.py

@@ -19,6 +19,8 @@
 import numpy as np
 import xarray as xr
 
+from disdrodb.constants import DIAMETER_DIMENSION, VELOCITY_DIMENSION
+
 
 def filter_diameter_bins(ds, minimum_diameter=None, maximum_diameter=None):
     """
@@ -29,10 +31,10 @@ def filter_diameter_bins(ds, minimum_diameter=None, maximum_diameter=None):
     ds : xarray.Dataset
         The dataset containing diameter bin data.
     minimum_diameter : float, optional
-        The minimum diameter to include in the filter, in millimeters.
+        The minimum diameter to be included, in millimeters.
         Defaults to the minimum value in `ds["diameter_bin_lower"]`.
     maximum_diameter : float, optional
-        The maximum diameter to include in the filter, in millimeters.
+        The maximum diameter to be included, in millimeters.
         Defaults to the maximum value in `ds["diameter_bin_upper"]`.
 
     Returns
@@ -40,22 +42,28 @@ def filter_diameter_bins(ds, minimum_diameter=None, maximum_diameter=None):
     xarray.Dataset
         The filtered dataset containing only the specified diameter bins.
     """
+    # Put data into memory
+    ds["diameter_bin_lower"] = ds["diameter_bin_lower"].compute()
+    ds["diameter_bin_upper"] = ds["diameter_bin_upper"].compute()
+
     # Initialize default arguments
     if minimum_diameter is None:
         minimum_diameter = ds["diameter_bin_lower"].min().item()
     if maximum_diameter is None:
         maximum_diameter = ds["diameter_bin_upper"].max().item()
-    # Select valid bins
+
+    # Select bins which overlap the specified diameters
     valid_indices = np.logical_and(
-        ds["diameter_bin_lower"] >= minimum_diameter,
-        ds["diameter_bin_upper"] <= maximum_diameter,
-    )
-    ds = ds.isel({"diameter_bin_center": valid_indices})
-    # Update history
-    history = ds.attrs.get("history", "")
-    ds.attrs["history"] = (
-        history + f" Selected drops with diameters between {minimum_diameter} and {maximum_diameter} mm \n"
+        ds["diameter_bin_upper"] > minimum_diameter,
+        ds["diameter_bin_lower"] < maximum_diameter,
     )
+
+    # Select bins with diameter values entirely inside the specified min/max values
+    # valid_indices = np.logical_and(
+    #     ds["diameter_bin_lower"] >= minimum_diameter,
+    #     ds["diameter_bin_upper"] <= maximum_diameter,
+    # )
+    ds = ds.isel({DIAMETER_DIMENSION: valid_indices})
     return ds
 
 
@@ -79,22 +87,28 @@ def filter_velocity_bins(ds, minimum_velocity=0, maximum_velocity=12):
     xarray.Dataset
         The filtered dataset containing only the specified velocity bins.
     """
+    # Put data into memory
+    ds["velocity_bin_lower"] = ds["velocity_bin_lower"].compute()
+    ds["velocity_bin_upper"] = ds["velocity_bin_upper"].compute()
+
     # Initialize default arguments
     if minimum_velocity is None:
         minimum_velocity = ds["velocity_bin_lower"].min().item()
     if maximum_velocity is None:
         maximum_velocity = ds["velocity_bin_upper"].max().item()
-    # Select valid bins
+
+    # Select bins which overlap the specified velocities
     valid_indices = np.logical_and(
-        ds["velocity_bin_lower"] >= minimum_velocity,
-        ds["velocity_bin_upper"] <= maximum_velocity,
-    )
-    ds = ds.isel({"velocity_bin_center": valid_indices})
-    # Update history
-    history = ds.attrs.get("history", "")
-    ds.attrs["history"] = (
-        history + f" Selected drops with fall velocity between {minimum_velocity} and {maximum_velocity} m/s \n"
+        ds["velocity_bin_upper"] > minimum_velocity,
+        ds["velocity_bin_lower"] < maximum_velocity,
     )
+
+    # Select bins with velocity values entirely inside the specified min/max values
+    # valid_indices = np.logical_and(
+    #     ds["velocity_bin_lower"] >= minimum_velocity,
+    #     ds["velocity_bin_upper"] <= maximum_velocity,
+    # )
+    ds = ds.isel({VELOCITY_DIMENSION: valid_indices})
     return ds