disdrodb 0.0.21__py3-none-any.whl → 0.1.0__py3-none-any.whl

disdrodb/l0/l0a_processing.py

@@ -19,7 +19,6 @@
 """Functions to process raw text files into DISDRODB L0A Apache Parquet."""
 
 
-import inspect
 import logging
 import os
 from typing import Union
@@ -39,7 +38,6 @@ from disdrodb.utils.directories import create_directory, remove_if_exists
 
 # Logger
 from disdrodb.utils.logger import (
-    log_debug,
     log_error,
     log_info,
     log_warning,
@@ -55,7 +53,7 @@ pd.set_option("mode.chained_assignment", None)  # Avoid SettingWithCopyWarning
 #### Raw file readers
 
 
-def _preprocess_reader_kwargs(reader_kwargs: dict) -> dict:
+def preprocess_reader_kwargs(reader_kwargs: dict) -> dict:
     """Preprocess arguments required to read raw text file into Pandas.
 
     Parameters
@@ -86,10 +84,20 @@ def _preprocess_reader_kwargs(reader_kwargs: dict) -> dict:
     return reader_kwargs
 
 
-def read_raw_file(
+def check_matching_column_number(df, column_names):
+    """Check the number of columns in the dataframe matches the length of column names."""
+    n_columns = len(df.columns)
+    n_expected_columns = len(column_names)
+    if n_columns != n_expected_columns:
+        msg = f"The dataframe has {n_columns} columns, while {n_expected_columns} are expected !."
+        raise ValueError(msg)
+
+
+def read_raw_text_file(
     filepath: str,
     column_names: list,
     reader_kwargs: dict,
+    logger=None,  # noqa
 ) -> pd.DataFrame:
     """Read a raw file into a dataframe.
 
@@ -100,7 +108,12 @@ def read_raw_file(
     column_names : list
         Column names.
     reader_kwargs : dict
-        Pandas pd.read_csv arguments.
+        Pandas ``pd.read_csv`` arguments.
+    logger : logging.Logger
+        Logger object.
+        The default is ``None``.
+        If ``None``, the logger is created using the module name.
+        If ``logger`` is passed, it will be used to log messages.
 
     Returns
     -------
@@ -108,7 +121,7 @@ def read_raw_file(
         Pandas dataframe.
     """
     # Preprocess reader_kwargs
-    reader_kwargs = _preprocess_reader_kwargs(reader_kwargs)
+    reader_kwargs = preprocess_reader_kwargs(reader_kwargs)
 
     # Enforce all raw files columns with dtype = 'object'
     dtype = "object"
@@ -117,8 +130,17 @@ def read_raw_file(
     try:
         df = pd.read_csv(filepath, names=column_names, dtype=dtype, **reader_kwargs)
     except pd.errors.EmptyDataError:
-        msg = f" - Is empty, skip file: {filepath}"
-        log_warning(logger=logger, msg=msg, verbose=False)
+        msg = f"The following file is empty: {filepath}"
+        raise ValueError(msg)
+
+    # Check the dataframe is not empty
+    if len(df.index) == 0:
+        msg = f"The following file is empty: {filepath}"
+        raise ValueError(msg)
+
+    # Check dataframe column number matches columns_names
+    if column_names is not None:
+        check_matching_column_number(df, column_names)
 
     # Return dataframe
     return df
@@ -128,45 +150,19 @@ def read_raw_file(
 #### L0A checks and homogenization
 
 
-def _check_df_sanitizer_fun(df_sanitizer_fun):
-    """Check the argument of df_sanitizer_fun is only df."""
-    if df_sanitizer_fun is None:
-        return None
-    if not callable(df_sanitizer_fun):
-        raise ValueError("'df_sanitizer_fun' must be a function.")
-    if not np.all(np.isin(inspect.getfullargspec(df_sanitizer_fun).args, ["df"])):
-        raise ValueError("The `df_sanitizer_fun` must have only `df` as input argument!")
-
-
-def _check_not_empty_dataframe(df, verbose=False):
-    if len(df.index) == 0:
-        msg = " - The file is empty and has been skipped."
-        log_error(logger=logger, msg=msg, verbose=False)
-        raise ValueError(msg)
-
-
-def _check_matching_column_number(df, column_names, verbose=False):
-    n_columns = len(df.columns)
-    n_expected_columns = len(column_names)
-    if n_columns != n_expected_columns:
-        msg = f" - The dataframe has {n_columns} columns, while {n_expected_columns} are expected !."
-        log_error(logger, msg, verbose)
-        raise ValueError(msg)
-
-
-def remove_rows_with_missing_time(df: pd.DataFrame, verbose: bool = False):
-    """Remove dataframe rows where the "time" is NaT.
+def remove_rows_with_missing_time(df: pd.DataFrame, logger=logger, verbose: bool = False):
+    """Remove dataframe rows where the ``"time"`` is ``NaT``.
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         Input dataframe.
     verbose : bool
-        Whether to verbose the processing.
+        Whether to verbose the processing. The default is ``False``.
 
     Returns
     -------
-    pd.DataFrame
+    pandas.DataFrame
         Dataframe with valid timesteps.
     """
     # Get the number of rows of the dataframe
@@ -175,32 +171,31 @@ def remove_rows_with_missing_time(df: pd.DataFrame, verbose: bool = False):
     df = df.dropna(subset="time", axis=0)
     # If no valid timesteps, raise error
     if len(df.index) == 0:
-        msg = " - There are not valid timestep."
-        log_error(logger=logger, msg=msg, verbose=False)
+        msg = "There are not valid timestep."
         raise ValueError(msg)
     # Otherwise, report the number of invalid timesteps
     n_invalid_timesteps = n_rows - len(df)
     if n_invalid_timesteps > 0:
-        msg = f" - {n_invalid_timesteps} rows had invalid timesteps and were discarded."
+        msg = f"{n_invalid_timesteps} rows had invalid timesteps and were discarded."
         log_warning(logger=logger, msg=msg, verbose=verbose)
     return df
 
 
-def remove_duplicated_timesteps(df: pd.DataFrame, verbose: bool = False):
+def remove_duplicated_timesteps(df: pd.DataFrame, logger=None, verbose: bool = False):
     """Remove duplicated timesteps.
 
     It keep only the first timestep occurrence !
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         Input dataframe.
     verbose : bool
-        Whether to verbose the processing.
+        Whether to verbose the processing. The default is ``False``.
 
     Returns
     -------
-    pd.DataFrame
+    pandas.DataFrame
         Dataframe with valid unique timesteps.
     """
     values, counts = np.unique(df["time"], return_counts=True)
@@ -208,11 +203,13 @@ def remove_duplicated_timesteps(df: pd.DataFrame, verbose: bool = False):
     values_duplicates = values[idx_duplicates].astype("M8[s]")
     # If there are duplicated timesteps
     if len(values_duplicates) > 0:
+        # TODO: raise error if duplicated timesteps have different values !
+
         # Drop duplicated timesteps (keeping the first occurrence)
         df = df.drop_duplicates(subset="time", keep="first")
         # Report the values of duplicated timesteps
         msg = (
-            f" - The following timesteps occurred more than once: {values_duplicates}. Only the first occurrence"
+            f"The following timesteps occurred more than once: {values_duplicates}. Only the first occurrence"
             " selected."
         )
         log_warning(logger=logger, msg=msg, verbose=verbose)
@@ -225,13 +222,12 @@ def drop_timesteps(df, timesteps):
     # Check there are row left
     if len(df) == 0:
         msg = "No rows left after removing problematic timesteps. Maybe you need to adjust the issue YAML file."
-        log_warning(logger=logger, msg=msg, verbose=False)
         raise ValueError(msg)
     return df
 
 
 def drop_time_periods(df, time_periods):
-    """Drop problematic time_period."""
+    """Drop problematic time periods."""
     for time_period in time_periods:
         if len(df) > 0:
             start_time = time_period[0]
@@ -240,25 +236,26 @@ def drop_time_periods(df, time_periods):
     # Check there are row left
     if len(df) == 0:
         msg = "No rows left after removing problematic time_periods. Maybe you need to adjust the issue YAML file."
-        log_warning(logger=logger, msg=msg, verbose=False)
         raise ValueError(msg)
 
     return df
 
 
-def remove_issue_timesteps(df, issue_dict, verbose=False):
+def remove_issue_timesteps(df, issue_dict, logger=None, verbose=False):
     """Drop dataframe rows with timesteps listed in the issue dictionary.
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         Input dataframe.
     issue_dict : dict
-        Issue dictionary
+        Issue dictionary.
+    verbose : bool
+        Whether to verbose the processing. The default is ``False``.
 
     Returns
     -------
-    pd.DataFrame
+    pandas.DataFrame
         Dataframe with problematic timesteps removed.
 
     """
@@ -286,24 +283,21 @@ def remove_issue_timesteps(df, issue_dict, verbose=False):
     return df
 
 
-def cast_column_dtypes(df: pd.DataFrame, sensor_name: str, verbose: bool = False) -> pd.DataFrame:
-    """Convert 'object' dataframe columns into DISDRODB L0A dtype standards.
+def cast_column_dtypes(df: pd.DataFrame, sensor_name: str) -> pd.DataFrame:
+    """Convert ``'object'`` dataframe columns into DISDRODB L0A dtype standards.
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         Input dataframe.
     sensor_name : str
         Name of the sensor.
-    verbose : bool
-        Whether to verbose the processing.
 
     Returns
     -------
-    pd.DataFrame
+    pandas.DataFrame
         Dataframe with corrected columns types.
     """
-
     # Cast dataframe to dtypes
     dtype_dict = get_l0a_dtype(sensor_name)
     # Ensure time column is saved with seconds resolution
@@ -321,26 +315,23 @@ def cast_column_dtypes(df: pd.DataFrame, sensor_name: str, verbose: bool = False
             df[column] = df[column].astype(dtype_dict[column])
         except ValueError as e:
             msg = f"ValueError: The column {column} has {e}"
-            log_error(logger=logger, msg=msg, verbose=False)
             raise ValueError(msg)
     return df
 
 
-def coerce_corrupted_values_to_nan(df: pd.DataFrame, sensor_name: str, verbose: bool = False) -> pd.DataFrame:
-    """Coerce corrupted values in dataframe numeric columns to np.nan.
+def coerce_corrupted_values_to_nan(df: pd.DataFrame, sensor_name: str) -> pd.DataFrame:
+    """Coerce corrupted values in dataframe numeric columns to ``np.nan``.
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         Input dataframe.
     sensor_name : str
         Name of the sensor.
-    verbose : bool
-        Whether to verbose the processing.
 
     Returns
     -------
-    pd.DataFrame
+    pandas.DataFrame
         Dataframe with string columns without corrupted values.
     """
     # Cast dataframe to dtypes
@@ -359,21 +350,19 @@ def coerce_corrupted_values_to_nan(df: pd.DataFrame, sensor_name: str, verbose:
     return df
 
 
-def strip_string_spaces(df: pd.DataFrame, sensor_name: str, verbose: bool = False) -> pd.DataFrame:
+def strip_string_spaces(df: pd.DataFrame, sensor_name: str) -> pd.DataFrame:
     """Strip leading/trailing spaces from dataframe string columns.
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         Input dataframe.
     sensor_name : str
         Name of the sensor.
-    verbose : bool
-        Whether to verbose the processing.
 
     Returns
     -------
-    pd.DataFrame
+    pandas.DataFrame
         Dataframe with string columns without leading/trailing spaces.
     """
     # Cast dataframe to dtypes
@@ -390,13 +379,13 @@ def strip_string_spaces(df: pd.DataFrame, sensor_name: str, verbose: bool = Fals
             try:
                 df[column] = df[column].str.strip()
             except AttributeError:
-                msg = f"AttributeError: The column {column} is not a string/object dtype."
-                log_error(logger=logger, msg=msg, verbose=False)
+                msg = f"The column {column} is not a string/object dtype."
                 raise AttributeError(msg)
     return df
 
 
-def _strip_delimiter(string):
+def strip_delimiter(string):
+    """Remove the first and last delimiter occurrence from a string."""
     if not isinstance(string, str):
         return string
     split_str = infer_split_str(string=string)
@@ -415,12 +404,12 @@ def strip_delimiter_from_raw_arrays(df):
     available_fields = list(df.columns[np.isin(df.columns, possible_fields)])
     # Loop over the fields and strip away the delimiter
     for field in available_fields:
-        df[field] = df[field].apply(_strip_delimiter)
+        df[field] = df[field].apply(strip_delimiter)
     # Return the dataframe
     return df
 
 
-def _is_not_corrupted(string):
+def is_raw_array_string_not_corrupted(string):
     """Check if the raw array is corrupted."""
     if not isinstance(string, str):
         return False
@@ -445,32 +434,32 @@ def remove_corrupted_rows(df):
     # Loop over the fields and remove corrupted ones
     for field in available_fields:
         if len(df) != 0:
-            df = df[df[field].apply(_is_not_corrupted)]
+            df = df[df[field].apply(is_raw_array_string_not_corrupted)]
     # Check if there are rows left
     if len(df) == 0:
         raise ValueError("No remaining rows after data corruption checks.")
     # If only one row available, raise also error
     if len(df) == 1:
-        raise ValueError("Only 1 row remains after data corruption checks. Check the file.")
+        raise ValueError("Only 1 row remains after data corruption checks. Check the raw file and maybe delete it.")
     # Return the dataframe
     return df
 
 
-def replace_nan_flags(df, sensor_name, verbose):
-    """Set values corresponding to nan_flags to np.nan.
+def replace_nan_flags(df, sensor_name, logger=None, verbose=False):
+    """Set values corresponding to ``nan_flags`` to ``np.nan``.
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         Input dataframe.
     sensor_name : str
         Name of the sensor.
     verbose : bool
-        Whether to verbose the processing.
+        Whether to verbose the processing. The default is ``False``.
 
     Returns
     -------
-    pd.DataFrame
+    pandas.DataFrame
         Dataframe without nan_flags values.
     """
     # Get dictionary of nan flags
@@ -486,26 +475,26 @@ def replace_nan_flags(df, sensor_name, verbose):
             if n_nan_flags_values > 0:
                 msg = f"In variable {var}, {n_nan_flags_values} values were nan_flags and were replaced to np.nan."
                 log_info(logger=logger, msg=msg, verbose=verbose)
-                df[var][is_a_nan_flag] = np.nan
+                df.loc[is_a_nan_flag, var] = np.nan
     # Return dataframe
     return df
 
 
-def set_nan_outside_data_range(df, sensor_name, verbose):
-    """Set values outside the data range as np.nan.
+def set_nan_outside_data_range(df, sensor_name, logger=None, verbose=False):
+    """Set values outside the data range as ``np.nan``.
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         Input dataframe.
     sensor_name : str
         Name of the sensor.
     verbose : bool
-        Whether to verbose the processing.
+        Whether to verbose the processing. The default is ``False``.
 
     Returns
     -------
-    pd.DataFrame
+    pandas.DataFrame
         Dataframe without values outside the expected data range.
     """
     # Get dictionary of data_range
@@ -530,21 +519,21 @@ def set_nan_outside_data_range(df, sensor_name, verbose):
     return df
 
 
-def set_nan_invalid_values(df, sensor_name, verbose):
-    """Set invalid (class) values to np.nan.
+def set_nan_invalid_values(df, sensor_name, logger=None, verbose=False):
+    """Set invalid (class) values to ``np.nan``.
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         Input dataframe.
     sensor_name : str
         Name of the sensor.
     verbose : bool
-        Whether to verbose the processing.
+        Whether to verbose the processing. The default is ``False``.
 
     Returns
     -------
-    pd.DataFrame
+    pandas.DataFrame
         Dataframe without invalid values.
     """
     # Get dictionary of valid values
@@ -566,14 +555,12 @@ def set_nan_invalid_values(df, sensor_name, verbose):
     return df
 
 
-def process_raw_file(
-    filepath,
-    column_names,
-    reader_kwargs,
-    df_sanitizer_fun,
+def sanitize_df(
+    df,
     sensor_name,
     verbose=True,
-    issue_dict={},
+    issue_dict=None,
+    logger=None,
 ):
     """Read and parse a raw text files into a L0A dataframe.
 
@@ -581,63 +568,41 @@ def process_raw_file(
     ----------
     filepath : str
         File path
-    column_names : list
-        Columns names.
-    reader_kwargs : dict
-         Pandas `read_csv` arguments.
-    df_sanitizer_fun : object, optional
-        Sanitizer function to format the datafame.
     sensor_name : str
         Name of the sensor.
     verbose : bool
-        Whether to verbose the processing.
-        The default is True
+        Whether to verbose the processing. The default is ``True``.
     issue_dict : dict
         Issue dictionary providing information on timesteps to remove.
-        The default is an empty dictionary {}.
-        Valid issue_dict key are 'timesteps' and 'time_periods'.
+        The default is an empty dictionary ``{}``.
+        Valid issue_dict key are ``'timesteps'`` and ``'time_periods'``.
         Valid issue_dict values are list of datetime64 values (with second accuracy).
-        To correctly format and check the validity of the issue_dict, use
-        the disdrodb.l0.issue.check_issue_dict function.
+        To correctly format and check the validity of the ``issue_dict``, use
+        the ``disdrodb.l0.issue.check_issue_dict`` function.
 
     Returns
     -------
-    pd.DataFrame
+    pandas.DataFrame
         Dataframe
     """
-    _check_df_sanitizer_fun(df_sanitizer_fun)
-
-    # Read the data
-    df = read_raw_file(
-        filepath=filepath,
-        column_names=column_names,
-        reader_kwargs=reader_kwargs,
-    )
-
-    # - Check if file empty
-    _check_not_empty_dataframe(df=df, verbose=verbose)
-
-    # - Check dataframe column number matches columns_names
-    _check_matching_column_number(df, column_names, verbose=False)
-
-    # - Sanitize the dataframe with a custom function
-    if df_sanitizer_fun is not None:
-        df = df_sanitizer_fun(df)
+    # Define the issue dictionary
+    # - If None, set to empty dictionary
+    issue_dict = {} if issue_dict is None else issue_dict
 
     # - Remove rows with time NaT
-    df = remove_rows_with_missing_time(df, verbose=verbose)
+    df = remove_rows_with_missing_time(df, logger=logger, verbose=verbose)
 
     # - Remove duplicated timesteps
-    df = remove_duplicated_timesteps(df, verbose=verbose)
+    df = remove_duplicated_timesteps(df, logger=logger, verbose=verbose)
 
     # - Filter out problematic tiemsteps reported in the issue YAML file
-    df = remove_issue_timesteps(df, issue_dict=issue_dict, verbose=verbose)
+    df = remove_issue_timesteps(df, issue_dict=issue_dict, logger=logger, verbose=verbose)
 
     # - Coerce numeric columns corrupted values to np.nan
-    df = coerce_corrupted_values_to_nan(df, sensor_name=sensor_name, verbose=verbose)
+    df = coerce_corrupted_values_to_nan(df, sensor_name=sensor_name)
 
     # - Strip trailing/leading space from string columns
-    df = strip_string_spaces(df, sensor_name=sensor_name, verbose=verbose)
+    df = strip_string_spaces(df, sensor_name=sensor_name)
 
     # - Strip first and last delimiter from the raw arrays
     df = strip_delimiter_from_raw_arrays(df)
@@ -646,16 +611,19 @@ def process_raw_file(
     df = remove_corrupted_rows(df)
 
     # - Cast dataframe to dtypes
-    df = cast_column_dtypes(df, sensor_name=sensor_name, verbose=verbose)
+    df = cast_column_dtypes(df, sensor_name=sensor_name)
 
     # - Replace nan flags values with np.nans
-    df = replace_nan_flags(df, sensor_name=sensor_name, verbose=verbose)
+    df = replace_nan_flags(df, sensor_name=sensor_name, logger=logger, verbose=verbose)
 
     # - Set values outside the data range to np.nan
-    df = set_nan_outside_data_range(df, sensor_name=sensor_name, verbose=verbose)
+    df = set_nan_outside_data_range(df, sensor_name=sensor_name, logger=logger, verbose=verbose)
 
     # - Replace invalid values with np.nan
-    df = set_nan_invalid_values(df, sensor_name=sensor_name, verbose=verbose)
+    df = set_nan_invalid_values(df, sensor_name=sensor_name, logger=logger, verbose=verbose)
+
+    # - Sort by time
+    df = df.sort_values("time")
 
     # ------------------------------------------------------.
     # - Check column names agrees to DISDRODB standards
@@ -677,23 +645,23 @@ def write_l0a(
     df: pd.DataFrame,
     filepath: str,
     force: bool = False,
+    logger=None,
     verbose: bool = False,
 ):
     """Save the dataframe into an Apache Parquet file.
 
     Parameters
     ----------
-    df : pd.DataFrame
+    df : pandas.DataFrame
         Input dataframe.
     filepath : str
         Output file path.
     force : bool, optional
         Whether to overwrite existing data.
-        If True, overwrite existing data into destination directories.
-        If False, raise an error if there are already data into destination directories. This is the default.
+        If ``True``, overwrite existing data into destination directories.
+        If ``False``, raise an error if there are already data into destination directories. This is the default.
     verbose : bool, optional
-        Whether to verbose the processing.
-        The default is False.
+        Whether to verbose the processing. The default is ``False``.
 
     Raises
     ------
@@ -702,7 +670,6 @@ def write_l0a(
     NotImplementedError
         The input dataframe can not be processed.
     """
-
     # -------------------------------------------------------------------------.
     # Create station directory if does not exist
     create_directory(os.path.dirname(filepath))
@@ -710,7 +677,7 @@ def write_l0a(
     # Check if the file already exists
     # - If force=True --> Remove it
     # - If force=False --> Raise error
-    remove_if_exists(filepath, force=force)
+    remove_if_exists(filepath, force=force, logger=logger)
 
     # -------------------------------------------------------------------------.
     # Define writing options
@@ -727,20 +694,18 @@ def write_l0a(
             row_group_size=row_group_size,
         )
         msg = f"The Pandas Dataframe has been written as an Apache Parquet file to {filepath}."
-        log_info(logger=logger, msg=msg, verbose=False)
+        log_info(logger=logger, msg=msg, verbose=verbose)
     except Exception as e:
-        msg = f" - The Pandas DataFrame cannot be written as an Apache Parquet file. The error is: \n {e}."
-        log_error(logger=logger, msg=msg, verbose=False)
+        msg = f"The Pandas DataFrame cannot be written as an Apache Parquet file. The error is: \n {e}."
         raise ValueError(msg)
     # -------------------------------------------------------------------------.
-    return None
 
 
-####---------------------------------------------------------------------------.
-#### L0A Utility
+####--------------------------------------------------------------------------.
+#### DISDRODB L0A product reader
 
 
-def concatenate_dataframe(list_df: list, verbose: bool = False) -> pd.DataFrame:
+def concatenate_dataframe(list_df: list, logger=None, verbose: bool = False) -> pd.DataFrame:
     """Concatenate a list of dataframes.
 
     Parameters
@@ -748,12 +713,12 @@ def concatenate_dataframe(list_df: list, verbose: bool = False) -> pd.DataFrame:
     list_df : list
         List of dataframes.
     verbose : bool, optional
-        If True, print messages.
-        If False, no print.
+        If ``True``, print messages.
+        If ``False``, no print.
 
     Returns
     -------
-    pd.DataFrame
+    pandas.DataFrame
         Concatenated dataframe.
 
     Raises
@@ -769,39 +734,111 @@ def concatenate_dataframe(list_df: list, verbose: bool = False) -> pd.DataFrame:
         return df
 
     # Log
-    msg = " - Concatenation of dataframes started."
-    log_info(logger, msg, verbose)
+    msg = "Concatenation of dataframes started."
+    log_info(logger=logger, msg=msg, verbose=verbose)
 
     # Concatenate the dataframe
     try:
         df = pd.concat(list_df, axis=0, ignore_index=True)
-
-        # Drop duplicated values
-        df = df.drop_duplicates(subset="time")
-
         # Sort by increasing time
         df = df.sort_values(by="time")
 
     except (AttributeError, TypeError) as e:
-        msg = f" - Can not concatenate the files. \n Error: {e}"
-        log_error(logger=logger, msg=msg, verbose=False)
+        msg = f"Can not concatenate the files. \n Error: {e}"
         raise ValueError(msg)
 
     # Log
-    msg = " - Concatenation of dataframes has finished."
-    log_info(logger, msg, verbose)
+    msg = "Concatenation of dataframes has finished."
+    log_info(logger=logger, msg=msg, verbose=verbose)
 
     # Return dataframe
     return df
 
 
-def read_raw_files(
+def _read_l0a(filepath: str, verbose: bool = False, logger=None, debugging_mode: bool = False) -> pd.DataFrame:
+    # Log
+    msg = f"Reading L0 Apache Parquet file at {filepath} started."
+    log_info(logger=logger, msg=msg, verbose=verbose)
+    # Open file
+    df = pd.read_parquet(filepath)
+    if debugging_mode:
+        df = df.iloc[0:100]
+    # Log
+    msg = f"Reading L0 Apache Parquet file at {filepath} ended."
+    log_info(logger=logger, msg=msg, verbose=verbose)
+    return df
+
+
+def read_l0a_dataframe(
+    filepaths: Union[str, list],
+    verbose: bool = False,
+    logger=None,
+    debugging_mode: bool = False,
+) -> pd.DataFrame:
+    """Read DISDRODB L0A Apache Parquet file(s).
+
+    Parameters
+    ----------
+    filepaths : str or list
+        Either a list or a single filepath.
+    verbose : bool
+        Whether to print detailed processing information into terminal.
+        The default is ``False``.
+    debugging_mode : bool
+        If ``True``, it reduces the amount of data to process.
+        If filepaths is a list, it reads only the first 3 files.
+        For each file it select only the first 100 rows.
+        The default is ``False``.
+
+    Returns
+    -------
+    pandas.DataFrame
+        L0A Dataframe.
+
+    """
+    from disdrodb.l0.l0a_processing import concatenate_dataframe
+
+    # ----------------------------------------
+    # Check filepaths validity
+    if not isinstance(filepaths, (list, str)):
+        raise TypeError("Expecting filepaths to be a string or a list of strings.")
+
+    # ----------------------------------------
+    # If filepath is a string, convert to list
+    if isinstance(filepaths, str):
+        filepaths = [filepaths]
+    # ---------------------------------------------------
+    # If debugging_mode=True, it reads only the first 3 filepaths
+    if debugging_mode:
+        filepaths = filepaths[0:3]  # select first 3 filepaths
+
+    # ---------------------------------------------------
+    # Define the list of dataframe
+    list_df = [
+        _read_l0a(filepath, verbose=verbose, logger=logger, debugging_mode=debugging_mode) for filepath in filepaths
+    ]
+
+    # Concatenate dataframe
+    df = concatenate_dataframe(list_df, logger=logger, verbose=verbose)
+
+    # Ensure time is in nanoseconds
+    df["time"] = df["time"].astype("M8[ns]")
+
+    # ---------------------------------------------------
+    # Return dataframe
+    return df
+
+
+####---------------------------------------------------------------------------.
+#### L0A Utility
+
+
+def read_raw_text_files(
     filepaths: Union[list, str],
-    column_names: list,
-    reader_kwargs: dict,
-    sensor_name: str,
-    verbose: bool,
-    df_sanitizer_fun: object = None,
+    reader,
+    sensor_name,
+    verbose=True,
+    logger=None,
 ) -> pd.DataFrame:
     """Read and parse a list for raw files into a dataframe.
 
@@ -809,20 +846,17 @@ def read_raw_files(
     ----------
     filepaths : Union[list,str]
         File(s) path(s)
-    column_names : list
-        Columns names.
-    reader_kwargs : dict
-         Pandas `read_csv` arguments.
+    reader:
+        DISDRODB reader function.
+        Format: reader(filepath, logger=None)
     sensor_name : str
         Name of the sensor.
     verbose : bool
-        Whether to verbose the processing.
-    df_sanitizer_fun : object, optional
-        Sanitizer function to format the datafame.
+        Whether to verbose the processing. The default is ``True``.
 
     Returns
     -------
-    pd.DataFrame
+    pandas.DataFrame
         Dataframe
 
     Raises
@@ -831,7 +865,6 @@ def read_raw_files(
         Input parameters can not be used or the raw file can not be processed.
 
     """
-
     # ------------------------------------------------------.
     # Check input list
     if isinstance(filepaths, str):
@@ -840,54 +873,50 @@ def read_raw_files(
         raise ValueError("'filepaths' must contains at least 1 filepath.")
 
     # ------------------------------------------------------.
-    ### - Loop over all raw files
+    # Loop over all raw files
     n_files = len(filepaths)
     processed_file_counter = 0
     list_skipped_files_msg = []
     list_df = []
     for filepath in filepaths:
+        # Try read the raw text file
         try:
-            # Try to process a raw file
-            df = process_raw_file(
-                filepath=filepath,
-                column_names=column_names,
-                reader_kwargs=reader_kwargs,
-                df_sanitizer_fun=df_sanitizer_fun,
+            df = reader(filepath, logger=logger)
+            # Sanitize the dataframe
+            df = sanitize_df(
+                df=df,
                 sensor_name=sensor_name,
+                logger=logger,
                 verbose=verbose,
             )
-
             # Append dataframe to the list
             list_df.append(df)
-
             # Update the logger
             processed_file_counter += 1
-            msg = f"{processed_file_counter} / {n_files} processed successfully. File name: {filepath}"
-            log_debug(logger=logger, msg=msg, verbose=verbose)
+            msg = f"Raw file '{filepath}' processed successfully ({processed_file_counter}/{n_files})."
+            log_info(logger=logger, msg=msg, verbose=verbose)
 
-        # If processing of raw file fails
+        # Skip the file if the processing fails
         except Exception as e:
             # Update the logger
-            msg = f" - {filepath} has been skipped. \n -- The error is: {e}."
-            log_warning(logger=logger, msg=msg, verbose=verbose)
+            msg = f"{filepath} has been skipped. The error is: {e}."
+            log_error(logger=logger, msg=msg, verbose=verbose)
             list_skipped_files_msg.append(msg)
 
     # Update logger
-    msg = f" - {len(list_skipped_files_msg)} of {n_files} have been skipped."
+    msg = f"{len(list_skipped_files_msg)} of {n_files} have been skipped."
     log_info(logger=logger, msg=msg, verbose=verbose)
-    logger.info("---")
-    logger.info(msg)
-    logger.info("---")
 
     ##----------------------------------------------------------------.
-    #### - Concatenate the dataframe
+    # Concatenate the dataframe
     if len(list_df) == 0:
-        raise ValueError(f"No dataframe to return. Impossible to parse {filepaths}.")
-    df = concatenate_dataframe(list_df, verbose=verbose)
-
-    # - Remove rows with duplicate timestep (keep the first)
-    df = df.drop_duplicates(subset=["time"], keep="first")
+        raise ValueError("Any raw file could be read!")
+    df = concatenate_dataframe(list_df, verbose=verbose, logger=logger)
 
     # ------------------------------------------------------.
+    # Enforce output time to be [ns]
+    # --> For compatibility with xarray
+    df["time"] = df["time"].astype("M8[ns]")
+
     # Return the dataframe
     return df