oceanarray 0.1.0__py3-none-any.whl

oceanarray/legacy/process_rodb.py

@@ -0,0 +1,369 @@
+import re
+from datetime import datetime
+from pathlib import Path
+
+import numpy as np
+import xarray as xr
+
+from oceanarray.logger import log_debug, log_info, log_warning
+
+from . import rodb
+
+DUMMY_VALUE = -9.99e-29  # adjust if needed
+
+
+def middle_percent(values, percent=95):
+    """Return the lower and upper bounds for the central `percent` of the data.
+
+    Parameters
+    ----------
+    values : array-like
+        Input data (1D array). NaNs will be ignored.
+    percent : float
+        Desired central percentage (e.g., 95 for middle 95%).
+
+    Returns
+    -------
+    tuple
+        (lower_bound, upper_bound)
+
+    """
+    values = np.asarray(values)
+    values = values[~np.isnan(values)]
+
+    if not 0 < percent < 100:
+        raise ValueError("percent must be between 0 and 100 (exclusive)")
+
+    tail = (100 - percent) / 2
+    lower = np.nanpercentile(values, tail)
+    upper = np.nanpercentile(values, 100 - tail)
+    return lower, upper
+
+
+def mean_of_middle_percent(values, percent=95):
+    """Compute the mean of values within the central `percent` of the data.
+
+    Parameters
+    ----------
+    values : array-like
+        Input data (1D array). NaNs will be ignored.
+    percent : float
+        Desired central percentage (e.g., 95 for middle 95%).
+
+    Returns
+    -------
+    float
+        Mean of values within the specified middle percentage.
+
+    """
+    values = np.asarray(values)
+    values = values[~np.isnan(values)]
+    lower, upper = middle_percent(values, percent)
+    filtered = values[(values >= lower) & (values <= upper)]
+    return np.mean(filtered)
+
+
+def std_of_middle_percent(values, percent=95):
+    """Compute the standard deviation of values within the central `percent` of the data.
+
+    Parameters
+    ----------
+    values : array-like
+        Input data (1D array). NaNs will be ignored.
+    percent : float
+        Desired central percentage (e.g., 95 for middle 95%).
+
+    Returns
+    -------
+    float
+        Standard deviation of values within the specified middle percentage.
+
+    """
+    values = np.asarray(values)
+    values = values[~np.isnan(values)]
+    lower, upper = middle_percent(values, percent)
+    filtered = values[(values >= lower) & (values <= upper)]
+    return np.std(filtered)
+
+
+def normalize_by_middle_percent(values, percent=95):
+    """Normalize a data series by the mean and standard deviation
+    of its central `percent` range.
+
+    Parameters
+    ----------
+    values : array-like
+        Input data (1D array). NaNs are ignored.
+    percent : float
+        Central percentage to define the 'middle' of the distribution (e.g., 95).
+
+    Returns
+    -------
+    array
+        Normalized array with the same shape as input.
+
+    """
+    values = np.asarray(values)
+    mask = ~np.isnan(values)
+    valid_values = values[mask]
+
+    if valid_values.size == 0:
+        return values  # return original if all NaNs
+
+    lower, upper = middle_percent(valid_values, percent)
+    middle_vals = valid_values[(valid_values >= lower) & (valid_values <= upper)]
+
+    mean_mid = np.mean(middle_vals)
+    std_mid = np.std(middle_vals)
+
+    if std_mid == 0:
+        raise ValueError(
+            "Standard deviation of middle percent is zero — normalization not possible."
+        )
+
+    normalized = (values - mean_mid) / std_mid
+    return normalized
+
+
+def normalize_dataset_by_middle_percent(ds, percent=95):
+    """Normalize all 1D data variables in an xarray Dataset that match the length of TIME,
+    using the mean and std over the central `percent` of each variable.
+
+    Parameters
+    ----------
+    ds : xarray.Dataset
+        Input dataset with a 'TIME' coordinate.
+    percent : float
+        Percentage of central values to define the middle (e.g., 95 for middle 95%).
+
+    Returns
+    -------
+    xarray.Dataset
+        New dataset with normalized data variables.
+
+    """
+    ds_norm = xr.Dataset(attrs=ds.attrs)
+    time_shape = ds["TIME"].shape
+
+    for var in ds.data_vars:
+        if ds[var].shape == time_shape:
+            norm_values = normalize_by_middle_percent(ds[var].values, percent)
+            ds_norm[var] = xr.DataArray(
+                norm_values,
+                coords=ds[var].coords,
+                dims=ds[var].dims,
+                attrs=ds[var].attrs,
+            )
+
+    # Retain TIME coordinate
+    ds_norm = ds_norm.assign_coords({"TIME": ds["TIME"]})
+    return ds_norm
+
+
+def trim_suggestion(ds, percent=95, threshold=6, vars_to_check=["T", "C", "P"]):
+    """Normalize dataset variables using the middle percentile and determine suggested
+    deployment start and end times where the normalized values are below a given threshold.
+
+    Parameters
+    ----------
+    ds : xarray.Dataset
+        Input dataset with a 'TIME' coordinate and 1D data variables.
+    percent : float
+        Percentage for middle-percent normalization (e.g., 95).
+    threshold : float
+        Absolute threshold on normalized values to consider as stable.
+    vars_to_check : list of str
+        List of variable names to consider for start/end detection.
+
+    Returns
+    -------
+    start_time : np.datetime64 or None
+        Suggested deployment start time.
+    end_time : np.datetime64 or None
+        Suggested deployment end time.
+
+    """
+    ds_norm = normalize_dataset_by_middle_percent(ds, percent=percent)
+
+    start_candidates = []
+    end_candidates = []
+
+    for var in vars_to_check:
+        if var not in ds_norm:
+            continue
+
+        arr = np.abs(ds_norm[var].values)
+        time_vals = ds_norm["TIME"].values
+
+        # First time index where normalized value drops below threshold
+        below_thresh = np.where(arr < threshold)[0]
+        if len(below_thresh) > 0:
+            first_below = time_vals[below_thresh[0]]
+            last_below = time_vals[below_thresh[-1]]
+            start_candidates.append(first_below)
+            end_candidates.append(last_below)
+        else:
+            print(f"Warning: {var} never drops below threshold {threshold}")
+
+    if not start_candidates or not end_candidates:
+        print("Could not determine suggested deployment period.")
+        return None, None
+
+    start_time = max(start_candidates)
+    end_time = min(end_candidates)
+
+    print(f"Suggested deployment start: {start_time}")
+    print(f"Suggested deployment end: {end_time}")
+    return start_time, end_time
+
+
+def stage2_trim(
+    ds: xr.Dataset,
+    deployment_start: datetime = None,
+    deployment_end: datetime = None,
+) -> xr.Dataset:
+    """Trim dataset to deployment period and fill time gaps with NaN.
+
+    Parameters
+    ----------
+    ds : xarray.Dataset
+        Input dataset with 'TIME' coordinate and variables like 'TEMP', 'CNDC', optionally 'PRES'.
+    deployment_start : datetime, optional
+        Start of the valid deployment period. If None, uses first timestamp in ds.
+    deployment_end : datetime, optional
+        End of the valid deployment period. If None, uses last timestamp in ds.
+
+    Returns
+    -------
+    xr.Dataset
+        Trimmed and gap-filled dataset.
+
+    """
+    if "TIME" not in ds:
+        raise ValueError("Dataset must contain 'TIME' coordinate")
+
+    if deployment_start is None:
+        deployment_start = ds.TIME.values[0]
+    if deployment_end is None:
+        deployment_end = ds.TIME.values[-1]
+
+    log_info(
+        "Trimming dataset to deployment period %s – %s",
+        deployment_start,
+        deployment_end,
+    )
+
+    # Trim to deployment period
+    ds_trimmed = ds.sel(TIME=slice(deployment_start, deployment_end))
+
+    # Infer sampling interval
+    time_diff = np.diff(ds_trimmed.TIME.values).astype("timedelta64[s]").astype(int)
+    if len(time_diff) == 0:
+        log_warning("Only one sample in dataset — no trimming or gap filling possible.")
+        return ds_trimmed
+
+    dt_seconds = int(np.median(time_diff))
+    sampling_interval = np.timedelta64(dt_seconds, "s")
+    log_debug("Inferred sampling interval: %d seconds", dt_seconds)
+
+    # Create regular time axis and reindex
+    time_start = ds_trimmed.TIME.values[0]
+    time_end = ds_trimmed.TIME.values[-1]
+    full_time = np.arange(time_start, time_end + sampling_interval, sampling_interval)
+
+    log_info("Interpolating to regular time base with %d steps", len(full_time))
+    ds_uniform = ds_trimmed.reindex(TIME=full_time, method=None)
+
+    # Log stats
+    for var in ["TEMP", "CNDC", "PRES"]:
+        if var in ds_uniform:
+            data = ds_uniform[var]
+            valid = np.isfinite(data.values)
+            log_info(
+                "%s: %d samples, mean = %.3f, std = %.3f, min = %.3f, max = %.3f",
+                var,
+                valid.sum(),
+                np.nanmean(data),
+                np.nanstd(data),
+                np.nanmin(data),
+                np.nanmax(data),
+            )
+
+    return ds_uniform, deployment_start, deployment_end
+
+
+def apply_microcat_calibration_from_txt(txt_file: str, use_file: str) -> xr.Dataset:
+    """Apply calibration offsets from a *.microcat.txt file to the original .use data file.
+
+    Parameters
+    ----------
+    txt_file : str
+        Path to the calibration log file (e.g., 'wb1_12_2015_005.microcat').
+    use_file : str
+        Path to the original '.use' file.
+
+    Returns
+    -------
+    ds : xr.Dataset
+        Dataset with calibrated temperature, conductivity, and pressure.
+
+    """
+    txt_file = Path(txt_file)
+    use_file = Path(use_file)
+
+    with open(txt_file, "r") as f:
+        text = f.read()
+
+    def get_offsets(variable):
+        pattern = rf"{variable}.*?:\s+([-0-9.]+)\s+([-0-9.]+)"
+        match = re.search(pattern, text)
+        if match:
+            return float(match.group(1)), float(match.group(2))
+        return None, None
+
+    def is_applied(var):
+        match = re.search(rf"Average {var} applied\?\s+([yn])", text, re.IGNORECASE)
+        return match and match.group(1).lower() == "y"
+
+    # Extract offsets and flags
+    c_pre, c_post = get_offsets("Conductivity")
+    t_pre, t_post = get_offsets("Temperature")
+    p_pre, p_post = get_offsets("Pressure")
+
+    apply_c = is_applied("conductivity")
+    apply_t = is_applied("temperature")
+    apply_p = is_applied("pressure")
+
+    # Load data using rodbload
+    ds = rodb.rodbload(use_file)
+
+    # Rename to standard names
+    # rename_map = {"T": "TEMP", "C": "CNDC", "P": "PRES"}
+    # ds = ds.rename({k: v for k, v in rename_map.items() if k in ds})
+
+    def apply_avg_offset(var, pre, post, flag):
+        if var not in ds or not flag or pre is None or post is None:
+            return ds.get(var, None)
+        offset = (pre + post) / 2
+        log_info(f"Applying average offset {offset:+.4f} to {var}")
+        return ds[var] + offset
+
+    # Apply calibration offsets
+    for var, pre, post, flag in [
+        ("T", t_pre, t_post, apply_t),
+        ("C", c_pre, c_post, apply_c),
+        ("P", p_pre, p_post, apply_p),
+    ]:
+        if var in ds:
+            ds[var] = apply_avg_offset(var, pre, post, flag)
+
+    # Replace dummy values
+    for var in ["T", "C", "P"]:
+        if var in ds:
+            ds[var] = ds[var].where(ds[var] != DUMMY_VALUE, np.nan)
+
+    ds.attrs["source_file"] = str(use_file)
+    ds.attrs["calibration_log"] = str(txt_file)
+    ds.attrs["comment"] = "Calibration offsets applied from microcat log file."
+
+    return ds

oceanarray/legacy/rodb.py

@@ -0,0 +1,330 @@
+"""rodbhead.py - Decode RO database keywords.
+
+Originally written in MATLAB by:
+  - G. Krahmann, IfM Kiel, Oct 1995 (v0.1.0)
+  - Updated for EPIC conventions: Feb 1996 (v0.1.1)
+  - Optimized and extended by multiple contributors through 2005
+  - Last change noted in MATLAB: C. Mertens, May 1997 (v0.1.4)
+  - Extensive extensions by D. Kieke (2000) and T. Kanzow (2005)
+
+Ported to Python by E Frajka-Williams, 2025
+"""
+
+import re
+from datetime import datetime
+from pathlib import Path
+
+import numpy as np
+import xarray as xr
+import yaml
+
+from oceanarray.logger import log_warning
+
+from .convertOS import parse_rodb_metadata
+
+REVERSE_KEYS = {
+    "mooring": "Mooring",
+    "serial_number": "SerialNumber",
+    "water_depth": "WaterDepth",
+    "instrdepth": "InstrDepth",  # if present in attrs
+    "latitude": "Latitude",  # if present in attrs
+    "longitude": "Longitude",  # if present in attrs
+    "start_time": ["Start_Date", "Start_Time"],  # split into date + time
+    "end_time": ["End_Date", "End_Time"],  # split into date + time
+    "columns": "Columns",
+}
+
+
+# in rodb.py
+def is_rodb_file(filepath: Path) -> bool:
+    """Check whether a file is RODB-style based on filename and/or content."""
+    return filepath.suffix.lower() in {".raw", ".use", ".microcat"}
+
+
+def parse_rodb_keys_file(filepath):
+    """Parse a rodb_keys.txt file with MATLAB-style lines into structured dicts.
+
+    Returns a dictionary with a list of entries under the 'RODB_KEYS' key.
+    """
+    entries = []
+    pattern = re.compile(r"^\s*'(?P<line>[^']+)';\s*\.\.\.\s*%?\s*(?P<comment>.*)?")
+
+    with open(filepath, "r") as f:
+        for line in f:
+            line = line.strip()
+            if not line.startswith("'"):
+                continue
+
+            match = pattern.match(line)
+            if not match:
+                continue
+            # Extract the line content and optional comment
+            print(line)
+            raw_fields = match.group("line").split()
+            if len(raw_fields) < 4:
+                continue  # not enough fields
+
+            key = raw_fields[0]
+            fmt = raw_fields[1]
+            count = int(raw_fields[2])
+            code = int(raw_fields[3])
+            comment = match.group("comment").strip() if match.group("comment") else None
+
+            entry = {
+                "key": key,
+                "format": fmt,
+                "count": count,
+                "code": code,
+            }
+            if comment:
+                entry["comment"] = comment
+
+            entries.append(entry)
+
+    return {"RODB_KEYS": entries}
+
+
+# Example usage
+if __name__ == "__main__":
+    infile = Path("rodb_keys.txt")  # Replace with your actual file path
+    rodb_keys_data = parse_rodb_keys_file(infile)
+
+    with open("rodb_keys.yaml", "w") as outfile:
+        yaml.dump(rodb_keys_data, outfile, sort_keys=False)
+
+
+# Load full RODB key metadata from YAML file
+RODB_KEYS_PATH = Path(__file__).parent.parent / "config" / "legacy" / "rodb_keys.yaml"
+with open(RODB_KEYS_PATH, "r") as f:
+    RODB_KEYS = yaml.safe_load(f)
+
+# Dummy placeholder values that indicate missing data
+DUMMY_VALUES = [-9999, -99.999, -999.999]
+
+
+def rodbload(filepath, variables: list[str] = None) -> xr.Dataset:
+    """Read a RODB .use or .raw file into an xarray.Dataset."""
+    filepath = Path(filepath)
+    with open(filepath, "r") as f:
+        lines = f.readlines()
+
+    header, data_start_index = parse_rodb_metadata(filepath)
+
+    default_dim = "N_MEASUREMENTS"
+
+    # Determine variables
+    var_string = header.get("COLUMNS", "YY:MM:DD:HH:T:C:P")
+    file_variables = var_string.split(":")
+    if variables is None:
+        variables = file_variables
+    col_indices = {var: i for i, var in enumerate(file_variables) if var in variables}
+
+    # Load numeric data
+    data = np.genfromtxt(lines[data_start_index:], dtype=float)
+    if data.ndim == 1:
+        data = data[np.newaxis, :]
+
+    for dummy in DUMMY_VALUES:
+        data[data == dummy] = np.nan
+
+    coords = {default_dim: np.arange(data.shape[0])}
+    data_vars = {var: (default_dim, data[:, col_indices[var]]) for var in variables}
+
+    if all(var in col_indices for var in ["YY", "MM", "DD", "HH"]):
+        y, m, d, h = (data[:, col_indices[v]] for v in ["YY", "MM", "DD", "HH"])
+        time = [
+            datetime(int(yy), int(mm), int(dd), int(hh), int((hh % 1) * 60))
+            for yy, mm, dd, hh in zip(y, m, d, h)
+        ]
+        coords["TIME"] = (default_dim, np.array(time, dtype="datetime64[s]"))
+    else:
+        # Optional: warn if TIME can't be constructed
+        missing = [v for v in ["YY", "MM", "DD", "HH"] if v not in col_indices]
+        log_warning("Could not create TIME coordinate. Missing: %s", ", ".join(missing))
+
+    ds = xr.Dataset(data_vars, coords=coords)
+
+    # Now convert header fields into attributes and variables
+    attrs = {}
+
+    # Global metadata
+    attrs["mooring"] = header.get("MOORING")
+    attrs["serial_number"] = header.get("SERIALNUMBER")
+    attrs["water_depth"] = int(header["WATERDEPTH"]) if "WATERDEPTH" in header else None
+
+    # Combine start and end date+time into ISO strings
+    if "START_DATE" in header and "START_TIME" in header:
+        attrs["start_time"] = f"{header['START_DATE']}T{header['START_TIME']}"
+    if "END_DATE" in header and "END_TIME" in header:
+        attrs["end_time"] = f"{header['END_DATE']}T{header['END_TIME']}"
+
+    # Convert InstrDepth, Latitude, Longitude to variables
+    if "INSTRDEPTH" in header:
+        ds["InstrDepth"] = header["INSTRDEPTH"] = float(header["INSTRDEPTH"])
+    if "LATITUDE" in header:
+        lat_str = header["LATITUDE"]
+        ds["Latitude"] = float(lat_str.split()[0]) + float(lat_str.split()[1]) / 60.0
+        if "S" in lat_str:
+            ds["Latitude"] *= -1
+    if "LONGITUDE" in header:
+        lon_str = header["LONGITUDE"]
+        ds["Longitude"] = float(lon_str.split()[0]) + float(lon_str.split()[1]) / 60.0
+        if "W" in lon_str:
+            ds["Longitude"] *= -1
+
+    # Add file reference and inferred columns
+    attrs["source_file"] = str(filepath)
+    attrs["columns"] = list(data_vars.keys())
+    # Promote TIME to an index if available
+    if "TIME" in ds.coords:
+        ds = ds.swap_dims({default_dim: "TIME"})
+
+    ds.attrs.update({k: v for k, v in attrs.items() if v is not None})
+    return ds
+
+
+NOMINAL_HEADER_ORDER = [
+    "Mooring",
+    "SerialNumber",
+    "WaterDepth",
+    "InstrDepth",
+    "Start_Date",
+    "Start_Time",
+    "End_Date",
+    "End_Time",
+    "Latitude",
+    "Longitude",
+    "Columns",
+]
+
+REVERSE_ATTR_KEYS = {
+    "Mooring": "mooring",
+    "SerialNumber": "serial_number",
+    "WaterDepth": "water_depth",
+    "InstrDepth": "instrdepth",
+    "Start_Date": "start_time",
+    "Start_Time": "start_time",
+    "End_Date": "end_time",
+    "End_Time": "end_time",
+    "Latitude": "latitude",
+    "Longitude": "longitude",
+    "Columns": "columns",
+}
+
+
+def format_latlon(value, is_lat=True):
+    deg = int(abs(value))
+    minutes = (abs(value) - deg) * 60
+    width = 2 if is_lat else 3
+    hemi = ("N" if value >= 0 else "S") if is_lat else ("E" if value >= 0 else "W")
+    return f"{deg:0{width}d} {minutes:06.3f} {hemi}"
+
+
+def rodbsave(filepath, ds: xr.Dataset, fmt=None):
+    """Save an xarray.Dataset to a RODB-style .use or .raw file.
+
+    Parameters
+    ----------
+    filepath : str or Path
+        Output file path.
+    ds : xarray.Dataset
+        Dataset to write. Expects time-series variables to be indexed by 'obs'.
+    fmt : str, optional
+        Format string passed to np.savetxt. If None, one is generated automatically.
+
+    """
+    filepath = Path(filepath)
+    attrs = ds.attrs.copy()
+
+    # Select time-series variables
+    timeseries_vars = attrs.get("columns")
+    if timeseries_vars is None:
+        timeseries_vars = [v for v in ds.data_vars if "obs" in ds[v].dims]
+
+    if fmt is None:
+        default_fmts = {
+            "YY": "%4d",
+            "MM": "%4d",
+            "DD": "%4d",
+            "HH": "%10.5f",
+            "T": "%9.4f",
+            "C": "%9.4f",
+            "P": "%7.1f",
+        }
+        fmt_parts = []
+        for var in timeseries_vars:
+            fmt_parts.append(default_fmts.get(var, "%10.4f"))
+        fmt = " ".join(fmt_parts)
+    written_keys = set()
+
+    with open(filepath, "w", newline="") as f:
+        for key in NOMINAL_HEADER_ORDER:
+            attr_key = REVERSE_ATTR_KEYS.get(key)
+
+            if (
+                key in ["Start_Date", "Start_Time"]
+                and "start_time" in attrs
+                and "Start_Date" not in written_keys
+            ):
+                try:
+                    dt = datetime.fromisoformat(attrs["start_time"])
+                except ValueError:
+                    dt = datetime.strptime(attrs["start_time"], "%Y/%m/%dT%H:%M")
+                f.write(f"{'Start_Date':<22} = {dt.strftime('%Y/%m/%d')}\n")
+                f.write(f"{'Start_Time':<22} = {dt.strftime('%H:%M')}\n")
+                written_keys.update(["Start_Date", "Start_Time"])
+                continue
+
+            if (
+                key in ["End_Date", "End_Time"]
+                and "end_time" in attrs
+                and "End_Date" not in written_keys
+            ):
+                try:
+                    dt = datetime.fromisoformat(attrs["end_time"])
+                except ValueError:
+                    dt = datetime.strptime(attrs["end_time"], "%Y/%m/%dT%H:%M")
+                f.write(f"{'End_Date':<22} = {dt.strftime('%Y/%m/%d')}\n")
+                f.write(f"{'End_Time':<22} = {dt.strftime('%H:%M')}\n")
+                written_keys.update(["End_Date", "End_Time"])
+                continue
+
+            if key == "Latitude" and "Latitude" not in written_keys:
+                val = ds.get("Latitude", None)
+                if val is not None:
+                    f.write(
+                        f"{'Latitude':<22} = {format_latlon(val.values.item(), is_lat=True)}\n"
+                    )
+                    written_keys.add("Latitude")
+                continue
+
+            if key == "Longitude" and "Longitude" not in written_keys:
+                val = ds.get("Longitude", None)
+                if val is not None:
+                    f.write(
+                        f"{'Longitude':<22} = {format_latlon(val.values.item(), is_lat=False)}\n"
+                    )
+                    written_keys.add("Longitude")
+                continue
+
+            if key == "Columns" and "Columns" not in written_keys:
+                f.write(f"{'Columns':<22} = " + ":".join(timeseries_vars) + "\n")
+                written_keys.add("Columns")
+                continue
+
+            if attr_key in attrs and key not in written_keys:
+                f.write(f"{key:<22} = {attrs[attr_key]}\n")
+                written_keys.add(key)
+
+            elif (
+                key in ds
+                and key not in written_keys
+                and (np.isscalar(ds[key].values) or ds[key].size == 1)
+            ):
+                val = ds[key].values.item()
+                f.write(f"{key:<22} = {val:.0f}\n")
+                written_keys.add(key)
+
+        # Write the data block
+        data_array = np.column_stack([ds[var].values for var in timeseries_vars])
+        np.savetxt(f, data_array, fmt=fmt)