misleep 0.3.0__py3-none-any.whl

misleep/io/annotation.py

@@ -0,0 +1,367 @@
+# -*- coding: UTF-8 -*-
+"""Annotation file I/O.
+
+The default MiSleep annotation format is a human readable ``.txt`` file
+with three sections:
+
+* ``==========Marker==========``     -- single time-point events
+* ``==========Start-End==========``  -- start/end events
+* ``==========Sleep state==========`` (or ``Sleep stage``) -- per-second states
+
+A bio-signal annotation format (first two lines are a header, then
+tab-separated state rows) is also supported through :func:`load_bio_anno`.
+"""
+
+import datetime
+import json
+import math
+from pathlib import Path
+
+from misleep.data import MiAnnotation
+from misleep.io.base import MiData  # noqa: F401 (kept for API symmetry)
+from misleep.logger import logger
+from misleep.utils.annotation import (
+    lst2group,
+    marker2mianno,
+    sleep_state2mianno,
+    start_end2mianno,
+)
+from misleep.utils.time_utils import transfer_time
+
+
+def load_misleep_anno(file_path, state_map=None):
+    """Load annotations from a MiSleep annotation file.
+
+    Parameters
+    ----------
+    file_path : str
+        Path of the annotation file.
+    state_map : dict, optional
+        Custom state code -> name mapping.
+
+    Returns
+    -------
+    MiAnnotation
+        The parsed annotation.
+    """
+    with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+        annotation = f.read().split("\n")
+
+    if annotation == [""]:
+        raise AssertionError("Empty")
+
+    try:
+        marker_idx = annotation.index("==========Marker==========")
+        start_end_idx = annotation.index("==========Start-End==========")
+        try:
+            sleep_state_idx = annotation.index("==========Sleep state==========")
+        except ValueError:
+            sleep_state_idx = annotation.index("==========Sleep stage==========")
+    except Exception:
+        raise AssertionError("Invalid")
+
+    marker = marker2mianno(annotation[marker_idx + 1: start_end_idx])
+    start_end = start_end2mianno(annotation[start_end_idx + 1: sleep_state_idx])
+    sleep_state = sleep_state2mianno(annotation[sleep_state_idx + 1:])
+
+    return MiAnnotation(sleep_state=sleep_state, start_end=start_end,
+                        marker=marker, state_map=state_map)
+
+
+def load_bio_anno(file_path):
+    """Load a bio-signal annotation file (tab-separated, 2-line header)."""
+    with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+        file = f.readlines()
+    file = file[2:]
+    state_list = [each.split("\t")[1] for each in file]
+    state_list = [[each] * 4 for each in state_list]
+    state_list = [item for each in state_list for item in each]
+    state_map = {
+        "AW": 3,
+        "QW": 3,
+        "NREM": 1,
+        "REMS": 2,
+    }
+    state_list = [state_map[each] for each in state_list]
+
+    return MiAnnotation(sleep_state=state_list, marker=[], start_end=[])
+
+
+def load_json_anno(file_path, state_map=None):
+    """Load a portable JSON annotation object.
+
+    Required key: ``sleep_state``. Optional keys are ``marker``,
+    ``start_end`` and ``state_map`` and mirror :class:`MiAnnotation`.
+    """
+    try:
+        payload = json.loads(Path(file_path).read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise ValueError(f"Invalid JSON annotation: {exc}") from exc
+    if not isinstance(payload, dict) or "sleep_state" not in payload:
+        raise ValueError("JSON annotation must be an object containing 'sleep_state'")
+    mapping = payload.get("state_map", state_map)
+    if mapping is not None:
+        mapping = {int(key): str(value) for key, value in mapping.items()}
+    return MiAnnotation(
+        sleep_state=[int(value) for value in payload["sleep_state"]],
+        marker=payload.get("marker", []),
+        start_end=payload.get("start_end", []),
+        state_map=mapping,
+    )
+
+
+def load_table_anno(file_path, state_map=None):
+    """Load per-second or interval sleep states from CSV/TSV.
+
+    A table needs a ``state``/``state_code``/``sleep_state`` column. With
+    ``start`` and ``end`` columns each row is an interval; otherwise every
+    row represents one second. Optional event rows use ``type=marker`` with
+    ``time`` and ``label``, or ``type=start_end`` with ``start/end/label``.
+    """
+    import pandas as pd
+
+    path = Path(file_path)
+    separator = "\t" if path.suffix.lower() == ".tsv" else ","
+    frame = pd.read_csv(path, sep=separator)
+    frame.columns = [str(column).strip().lower() for column in frame.columns]
+    mapping = state_map or {1: "NREM", 2: "REM", 3: "Wake", 4: "Init"}
+    mapping = {int(key): str(value) for key, value in mapping.items()}
+    reverse = {name.casefold(): code for code, name in mapping.items()}
+    state_column = next(
+        (name for name in ("state", "state_code", "sleep_state") if name in frame), None)
+
+    def state_code(value):
+        try:
+            code = int(value)
+        except (TypeError, ValueError):
+            code = reverse.get(str(value).strip().casefold())
+        if code not in mapping:
+            raise ValueError(f"Unknown sleep state {value!r}; expected {mapping}")
+        return code
+
+    kind = frame["type"].astype(str).str.lower() if "type" in frame else None
+    state_rows = frame if kind is None else frame[~kind.isin(["marker", "start_end", "event"])]
+    if state_column is None or state_rows.empty:
+        raise ValueError("Annotation table must contain sleep-state rows and a state column")
+    if {"start", "end"}.issubset(frame.columns):
+        duration = int(math.ceil(pd.to_numeric(state_rows["end"]).max()))
+        sleep_state = [max(mapping)] * duration
+        for _, row in state_rows.iterrows():
+            start = max(0, int(math.floor(float(row["start"]))))
+            end = min(duration, int(math.ceil(float(row["end"]))))
+            sleep_state[start:end] = [state_code(row[state_column])] * max(0, end - start)
+    else:
+        sleep_state = [state_code(value) for value in state_rows[state_column]]
+
+    marker, start_end = [], []
+    if kind is not None:
+        for index, row in frame[kind == "marker"].iterrows():
+            when = row.get("time", row.get("start"))
+            marker.append([float(when), str(row.get("label", "marker"))])
+        for index, row in frame[kind.isin(["start_end", "event"])].iterrows():
+            start_end.append([float(row["start"]), float(row["end"]),
+                              str(row.get("label", "event"))])
+    return MiAnnotation(sleep_state=sleep_state, marker=marker,
+                        start_end=start_end, state_map=mapping)
+
+
+def available_annotation_readers():
+    """Return annotation extensions understood by :func:`load_annotation`."""
+    return [".txt", ".json", ".csv", ".tsv"]
+
+
+def load_annotation(file_path, state_map=None):
+    """Load an annotation by extension, auto-detecting legacy TXT files."""
+    path = Path(file_path)
+    if not path.is_file():
+        raise FileNotFoundError(f"Annotation file not found: {path}")
+    suffix = path.suffix.lower()
+    if suffix == ".json":
+        return load_json_anno(path, state_map=state_map)
+    if suffix in (".csv", ".tsv"):
+        return load_table_anno(path, state_map=state_map)
+    if suffix == ".txt":
+        prefix = path.read_text(encoding="utf-8", errors="ignore")[:5]
+        return load_bio_anno(path) if prefix == "Start" else load_misleep_anno(path, state_map)
+    raise ValueError(f"Unsupported annotation extension {suffix!r}")
+
+
+def save_misleep_anno(mianno, midata, file_path):
+    """Write a :class:`MiAnnotation` to the MiSleep ``.txt`` format.
+
+    Parameters
+    ----------
+    mianno : MiAnnotation
+        The annotation to save.
+    midata : MiData
+        The associated data (used for the acquisition time).
+    file_path : str
+        Destination file.
+
+    Returns
+    -------
+    bool
+        True on success.
+    """
+    from misleep.utils.time_utils import second2time
+
+    ac_time = datetime.datetime.strptime(midata.time, "%Y%m%d-%H:%M:%S")
+
+    marker = [", ".join([
+        second2time(round(each[0], 3), ac_time=ac_time, ms=True), str(round(each[0], 3)), "1",
+        second2time(round(each[0], 3), ac_time=ac_time, ms=True), str(round(each[0], 3)), "0",
+        "1", each[1]
+    ]) for each in mianno.marker]
+
+    start_end_label = [", ".join([
+        second2time(round(each[0], 3), ac_time=ac_time, ms=True), str(round(each[0], 3)), "1",
+        second2time(round(each[1], 3), ac_time=ac_time, ms=True), str(round(each[1], 3)), "0",
+        "1", each[2]
+    ]) for each in mianno.start_end]
+
+    sleep_state = lst2group([[idx, each] for idx, each in enumerate(mianno.sleep_state)])
+    sleep_state = [", ".join([
+        second2time(each[0], ac_time=ac_time), str(each[0]), "1",
+        second2time(each[1], ac_time=ac_time), str(each[1]),
+        "0", str(each[2]), mianno.state_map[each[2]]
+    ]) for each in sleep_state]
+
+    if len(marker) > 0:
+        marker = [""] + marker
+    if len(start_end_label) > 0:
+        start_end_label = [""] + start_end_label
+
+    annos = [
+        "READ ONLY! DO NOT EDIT!\n4-INIT 3-Wake 2-REM 1-NREM",
+        "Save time: " + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+        "Acquisition time: " + ac_time.strftime("%Y-%m-%d %H:%M:%S"),
+        "==========Marker==========" + "\n".join(marker),
+        "==========Start-End==========" + "\n".join(start_end_label),
+        "==========Sleep stage==========", "\n".join(sleep_state)
+    ]
+
+    with open(file_path, "w", encoding="utf-8") as f:
+        f.write("\n".join(annos))
+    return True
+
+
+def transfer_result(mianno, ac_time):
+    """Convert a :class:`MiAnnotation` into analysis dataframes.
+
+    Produces per-hour sleep statistics (duration, bout count, average bout
+    length, percentage for every state), plus a 12h light/dark phase
+    summary. Also returns the raw marker/start-end dataframes.
+
+    Parameters
+    ----------
+    mianno : MiAnnotation
+        Annotation to transfer.
+    ac_time : datetime.datetime
+        Acquisition time of the recording.
+
+    Returns
+    -------
+    (df, analyse_df, start_end_df, marker_df) : tuple of pandas.DataFrame
+    """
+    import pandas as pd
+
+    marker = [[
+        transfer_time(ac_time, each[0], "%Y-%m-%d %H:%M:%S", ms=True),
+        each[0], each[1]] for each in mianno.marker]
+
+    start_end_label = [[
+        transfer_time(ac_time, each[0], "%Y-%m-%d %H:%M:%S", ms=True), each[0], 1,
+        transfer_time(ac_time, each[1], "%Y-%m-%d %H:%M:%S", ms=True), each[1], 0,
+        each[2], each[1] - each[0]
+    ] for each in mianno.start_end]
+
+    # Split the sleep state into per-hour groups
+    sleep_state = []
+    for each in range(int(len(mianno.sleep_state) / 3600) + 1):
+        temp_hour_label = mianno.sleep_state[each * 3600: (each + 1) * 3600]
+        if temp_hour_label != []:
+            sleep_state.append(temp_hour_label)
+
+    marker_sleep_state = []
+    for hour, label in enumerate(sleep_state):
+        hour_sleep_state = lst2group(
+            [idx + hour * 3600, each] for idx, each in enumerate(label))
+        marker_sleep_state += [[
+            transfer_time(ac_time, each[0], "%Y-%m-%d %H:%M:%S"), each[0], 1,
+            transfer_time(ac_time, each[1], "%Y-%m-%d %H:%M:%S"), each[1], 0,
+            each[2], mianno.state_map[each[2]], each[1] - each[0], hour
+        ] for each in hour_sleep_state]
+
+        marker_sleep_state += [[
+            transfer_time(ac_time, (hour + 1) * 3600, "%Y-%m-%d %H:%M:%S"), (hour + 1) * 3600, 1,
+            transfer_time(ac_time, (hour + 1) * 3600, "%Y-%m-%d %H:%M:%S"), (hour + 1) * 3600, 0,
+            5, "MARKER", "", ""]]
+
+    columns = ["start_time", "start_time_sec", "start_code",
+               "end_time", "end_time_sec", "end_code",
+               "state_code", "state", "bout_duration", "hour"]
+
+    df = pd.DataFrame(data=marker_sleep_state, columns=columns)
+
+    df["hour"] = df.apply(
+        lambda x: "" if x["state"] == "MARKER" else int(x["start_time_sec"] / 3600), axis=1)
+    analyse_df = pd.DataFrame()
+
+    temp_hour = list(set(list(df["hour"])))
+    temp_hour.remove("")
+    temp_hour = sorted(temp_hour)
+    analyse_df["date_time"] = [transfer_time(ac_time, each * 3600, "%Y-%m-%d %H:%M:%S")
+                               for each in temp_hour]
+
+    features = []
+    for each in temp_hour:
+        df_ = df[df["hour"] == each]
+        temp_lst = []
+        for phase in ["NREM", "REM", "Wake", "INIT"]:
+            _duration = df_[df_["state"] == phase]["bout_duration"].sum()
+            _bout = df_[df_["state"] == phase]["bout_duration"].count()
+            temp_lst += [_duration, _bout, round(_duration / _bout, 2) if _bout != 0 else 0,
+                         round(_duration / 3600, 4) * 100]
+        features.append(temp_lst)
+
+    analyse_df[["NREM_duration", "NREM_bout", "NREM_ave", "NREM_percentage",
+                "REM_duration", "REM_bout", "REM_ave", "REM_percentage",
+                "WAKE_duration", "WAKE_bout", "WAKE_ave", "WAKE_percentage",
+                "INIT_duration", "INIT_bout", "INIT_ave", "INIT_percentage"]] = features
+
+    analyse_df[["NREM_duration", "NREM_bout", "REM_duration", "REM_bout", "WAKE_duration",
+                "WAKE_bout", "INIT_duration", "INIT_bout"]] = analyse_df[
+        ["NREM_duration", "NREM_bout", "REM_duration", "REM_bout", "WAKE_duration",
+         "WAKE_bout", "INIT_duration", "INIT_bout"]].astype(int)
+
+    # 12-h light/dark phase summary (first hour treated as ZT0 by default)
+    try:
+        phase_data = pd.DataFrame()
+        phase_data["date_time"] = ["ZT0-ZT12", "ZT12-ZT24"]
+        for phase in ["NREM", "REM", "WAKE", "INIT"]:
+            duration_key = f"{phase}_duration"
+            bout_key = f"{phase}_bout"
+            ave_key = f"{phase}_ave"
+            pct_key = f"{phase}_percentage"
+            phase_data[duration_key] = [analyse_df[duration_key].iloc[:12].sum(),
+                                        analyse_df[duration_key].iloc[12:24].sum()]
+            phase_data[bout_key] = [analyse_df[bout_key].iloc[:12].sum(),
+                                    analyse_df[bout_key].iloc[12:24].sum()]
+            phase_data[ave_key] = phase_data.apply(
+                lambda x: x[duration_key] / x[bout_key] if x[bout_key] != 0 else 0, axis=1)
+            phase_data[pct_key] = phase_data.apply(lambda x: x[duration_key] / (3600 * 12), axis=1)
+
+        analyse_df = pd.concat([analyse_df, phase_data])
+        analyse_df.reset_index(inplace=True)
+    except Exception as e:
+        logger.warning(f"Could not compute light/dark phase summary: {e}")
+
+    start_end_df = pd.DataFrame(
+        start_end_label,
+        columns=["start_time", "start_time_sec", "start_code",
+                 "end_time", "end_time_sec", "end_code",
+                 "label", "bout_duration"])
+
+    marker_df = pd.DataFrame(marker, columns=["timestamp", "timestamp_sec", "label"])
+
+    return df, analyse_df, start_end_df, marker_df

misleep/io/array.py

@@ -0,0 +1,209 @@
+# -*- coding: UTF-8 -*-
+"""Portable NumPy and delimited-text signal formats.
+
+Plain arrays do not contain sampling frequency or acquisition-time metadata.
+MiSleep therefore uses a small JSON sidecar next to ``.npy``, ``.csv`` and
+``.tsv`` files.  For ``recording.npy`` either ``recording.npy.json`` (preferred)
+or ``recording.json`` is accepted::
+
+    {"sf": 256, "channels": ["EEG", "EMG"],
+     "time": "20240409-18:00:00", "channel_axis": 0}
+
+``.npz`` is self-contained and is the recommended NumPy interchange format.
+Loading never enables NumPy pickle support, so untrusted files cannot execute
+Python objects.
+"""
+
+from __future__ import annotations
+
+import datetime as _datetime
+import json
+from pathlib import Path
+
+import numpy as np
+
+from misleep.data import MiData
+from misleep.io.base import register_signal_reader, register_signal_writer
+
+_TIME_FORMAT = "%Y%m%d-%H:%M:%S"
+
+
+def _default_time(path: Path) -> str:
+    """Use the file modification time when acquisition time is unavailable."""
+    return _datetime.datetime.fromtimestamp(path.stat().st_mtime).strftime(_TIME_FORMAT)
+
+
+def _read_sidecar(path: Path) -> dict:
+    candidates = (path.with_suffix(path.suffix + ".json"), path.with_suffix(".json"))
+    for candidate in candidates:
+        if candidate.is_file():
+            try:
+                value = json.loads(candidate.read_text(encoding="utf-8"))
+            except (OSError, json.JSONDecodeError) as exc:
+                raise ValueError(f"Invalid metadata sidecar {candidate}: {exc}") from exc
+            if not isinstance(value, dict):
+                raise ValueError(f"Metadata sidecar {candidate} must contain a JSON object")
+            return value
+    return {}
+
+
+def _metadata_list(value, count: int, name: str, cast):
+    if np.isscalar(value):
+        values = [cast(value)] * count
+    else:
+        values = [cast(item) for item in value]
+    if len(values) != count:
+        raise ValueError(f"'{name}' has {len(values)} values for {count} signal channels")
+    return values
+
+
+def _to_signals(array, metadata: dict, path: Path):
+    data = np.asarray(array)
+    if data.dtype.kind not in "biufc":
+        raise ValueError(f"{path.name} must contain a numeric array, got dtype {data.dtype}")
+    data = np.squeeze(data)
+    if data.ndim == 1:
+        signals = [np.asarray(data)]
+    elif data.ndim == 2:
+        channel_axis = metadata.get("channel_axis")
+        if channel_axis is None:
+            channel_axis = 0 if data.shape[0] <= data.shape[1] else 1
+        if channel_axis not in (0, 1):
+            raise ValueError("'channel_axis' must be 0 or 1")
+        if channel_axis == 1:
+            data = data.T
+        signals = [np.asarray(row) for row in data]
+    else:
+        raise ValueError(
+            f"{path.name} must be a 1-D or 2-D signal array; got shape {data.shape}")
+    return signals
+
+
+def _build_midata(array, metadata: dict, path: Path) -> MiData:
+    if isinstance(array, list) and array and all(np.asarray(item).ndim == 1 for item in array):
+        signals = [np.asarray(item) for item in array]
+        if any(signal.dtype.kind not in "biufc" for signal in signals):
+            raise ValueError(f"{path.name} contains a non-numeric signal array")
+    else:
+        signals = _to_signals(array, metadata, path)
+    if any(signal.size == 0 for signal in signals):
+        raise ValueError(f"{path.name} contains an empty signal channel")
+    count = len(signals)
+    if "sf" not in metadata:
+        raise ValueError(
+            f"Sampling frequency is missing for {path.name}. Add 'sf' to "
+            f"{path.name}.json (for example: {{\"sf\": 256}}).")
+    sf = _metadata_list(metadata["sf"], count, "sf", float)
+    channels = metadata.get("channels", [f"ch{i + 1}" for i in range(count)])
+    channels = _metadata_list(channels, count, "channels", str)
+    time = str(metadata.get("time") or _default_time(path))
+    try:
+        _datetime.datetime.strptime(time, _TIME_FORMAT)
+    except ValueError as exc:
+        raise ValueError(f"'time' must use YYYYMMDD-HH:MM:SS, got {time!r}") from exc
+    return MiData(signals=signals, channels=channels, sf=sf, time=time,
+                  describe=str(metadata.get("describe", "")))
+
+
+def load_npy(data_path) -> MiData:
+    """Load a numeric ``.npy`` array plus its JSON metadata sidecar."""
+    path = Path(data_path)
+    try:
+        array = np.load(path, allow_pickle=False)
+    except ValueError as exc:
+        raise ValueError(
+            "Object/pickled NPY files are intentionally not loaded for security. "
+            "Use a numeric array with a JSON sidecar, or the self-contained NPZ format."
+        ) from exc
+    return _build_midata(array, _read_sidecar(path), path)
+
+
+def load_npz(data_path) -> MiData:
+    """Load MiSleep's safe, self-contained ``.npz`` interchange format."""
+    path = Path(data_path)
+    try:
+        archive = np.load(path, allow_pickle=False)
+        with archive:
+            keys = set(archive.files)
+            metadata = _read_sidecar(path)
+            for key in ("channels", "sf", "time", "describe", "channel_axis"):
+                if key in keys:
+                    value = archive[key]
+                    metadata[key] = value.item() if value.ndim == 0 else value.tolist()
+            signal_keys = sorted(
+                (key for key in keys if key.startswith("signal_")),
+                key=lambda key: int(key.split("_", 1)[1]),
+            )
+            if signal_keys:
+                signals = [np.asarray(archive[key]) for key in signal_keys]
+            elif "signals" in keys:
+                signals = _to_signals(archive["signals"], metadata, path)
+            else:
+                numeric = [key for key in archive.files
+                           if key not in {"sf", "channel_axis"}
+                           and np.asarray(archive[key]).dtype.kind in "biufc"]
+                if len(numeric) != 1:
+                    raise ValueError(
+                        "NPZ must contain 'signals', signal_0/signal_1 arrays, "
+                        "or exactly one numeric array")
+                signals = _to_signals(archive[numeric[0]], metadata, path)
+    except (OSError, ValueError, TypeError) as exc:
+        if isinstance(exc, ValueError) and "NPZ must" in str(exc):
+            raise
+        raise ValueError(f"Could not read NPZ signal file {path}: {exc}") from exc
+    return _build_midata(signals, metadata, path)
+
+
+def _load_delimited(data_path, separator: str) -> MiData:
+    import pandas as pd
+
+    path = Path(data_path)
+    frame = pd.read_csv(path, sep=separator)
+    if frame.empty:
+        raise ValueError(f"Signal table {path} is empty")
+    metadata = _read_sidecar(path)
+    time_column = next(
+        (name for name in frame.columns
+         if str(name).strip().lower() in {"time", "times", "second", "seconds", "timestamp"}),
+        None,
+    )
+    if time_column is not None and "sf" not in metadata:
+        time_values = pd.to_numeric(frame.pop(time_column), errors="raise").to_numpy()
+        steps = np.diff(time_values)
+        steps = steps[np.isfinite(steps) & (steps > 0)]
+        if not len(steps):
+            raise ValueError("Time column must contain increasing values to infer sampling frequency")
+        metadata["sf"] = float(1.0 / np.median(steps))
+    numeric = frame.apply(pd.to_numeric, errors="raise")
+    metadata.setdefault("channels", [str(name) for name in numeric.columns])
+    metadata["channel_axis"] = 1
+    return _build_midata(numeric.to_numpy(), metadata, path)
+
+
+def load_csv(data_path) -> MiData:
+    """Load a headered CSV; infer frequency from a time column if present."""
+    return _load_delimited(data_path, ",")
+
+
+def load_tsv(data_path) -> MiData:
+    """Load a headered tab-separated signal table."""
+    return _load_delimited(data_path, "\t")
+
+
+def write_npz(signals, channels, sf, time, npz_file) -> None:
+    """Write a pickle-free, self-contained NumPy archive."""
+    payload = {f"signal_{i}": np.asarray(signal) for i, signal in enumerate(signals)}
+    payload.update(channels=np.asarray(channels, dtype=str), sf=np.asarray(sf, dtype=float),
+                   time=np.asarray(str(time)))
+    np.savez_compressed(npz_file, **payload)
+
+
+for _extension, _reader in {
+    ".npy": load_npy,
+    ".npz": load_npz,
+    ".csv": load_csv,
+    ".tsv": load_tsv,
+}.items():
+    register_signal_reader(_extension, _reader)
+
+register_signal_writer(".npz", write_npz)