openfrenz 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,13 @@
1
+ recordings/
2
+ *.sqlite
3
+ *.sqlite-wal
4
+ *.sqlite-shm
5
+ *.db
6
+ target/
7
+ __pycache__/
8
+ .venv/
9
+ .DS_Store
10
+ sessions/
11
+ python/.venv/
12
+ *.egg-info/
13
+ .pytest_cache/
@@ -0,0 +1,30 @@
1
+ Metadata-Version: 2.5
2
+ Name: openfrenz
3
+ Version: 0.2.1
4
+ Summary: Load openfrenz session datasets (FRENZ brainband) and build aligned fixed-length windows for analysis
5
+ Project-URL: Repository, https://github.com/ljchang/openfrenz
6
+ Project-URL: Documentation, https://github.com/ljchang/openfrenz/blob/main/docs/CLI.md
7
+ License-Expression: MIT
8
+ Requires-Python: >=3.10
9
+ Requires-Dist: numpy>=1.26
10
+ Requires-Dist: pandas>=2.0
11
+ Requires-Dist: pyarrow>=15
12
+ Provides-Extra: test
13
+ Requires-Dist: pytest>=8; extra == 'test'
14
+ Description-Content-Type: text/markdown
15
+
16
+ # openfrenz (Python)
17
+
18
+ Load session folders written by `frenz export` and build aligned, fixed-length multimodal windows.
19
+
20
+ ```python
21
+ from openfrenz import load_session, WindowDataset, find_sessions
22
+
23
+ s = load_session("sessions/FRENZA12-1800000000")
24
+ t, eeg, segment = s.signal("eeg") # eeg: (4, n) raw counts
25
+ ds = WindowDataset(find_sessions("sessions"), window_s=10, hop_s=5,
26
+ streams=("eeg", "accel", "gyro", "ppg"), require=("eeg",), min_contact=3)
27
+ item = ds[0] # {"eeg": (4, 1250), "accel": (3, 500), "gyro": (3, 500), "ppg": (2, 250), "mask": {...}}
28
+ ```
29
+
30
+ See the [CLI guide](https://github.com/ljchang/openfrenz/blob/main/docs/CLI.md) for the session format.
@@ -0,0 +1,15 @@
1
+ # openfrenz (Python)
2
+
3
+ Load session folders written by `frenz export` and build aligned, fixed-length multimodal windows.
4
+
5
+ ```python
6
+ from openfrenz import load_session, WindowDataset, find_sessions
7
+
8
+ s = load_session("sessions/FRENZA12-1800000000")
9
+ t, eeg, segment = s.signal("eeg") # eeg: (4, n) raw counts
10
+ ds = WindowDataset(find_sessions("sessions"), window_s=10, hop_s=5,
11
+ streams=("eeg", "accel", "gyro", "ppg"), require=("eeg",), min_contact=3)
12
+ item = ds[0] # {"eeg": (4, 1250), "accel": (3, 500), "gyro": (3, 500), "ppg": (2, 250), "mask": {...}}
13
+ ```
14
+
15
+ See the [CLI guide](https://github.com/ljchang/openfrenz/blob/main/docs/CLI.md) for the session format.
@@ -0,0 +1,12 @@
1
+ """openfrenz: load FRENZ brainband session datasets and build ML-ready windows.
2
+
3
+ Sessions are folders written by the ``frenz export`` command (Parquet per stream plus
4
+ ``session.json``). Every sample has ``t``, its estimated acquisition time in Unix seconds on one
5
+ host clock, so streams are aligned. Values are raw device units; see ``session.meta``.
6
+ """
7
+
8
+ from .dataset import WindowDataset
9
+ from .session import SIGNALS, Session, find_sessions, load_session
10
+
11
+ __all__ = ["SIGNALS", "Session", "WindowDataset", "find_sessions", "load_session"]
12
+ __version__ = "0.2.1"
@@ -0,0 +1,125 @@
1
+ """Fixed-length, time-aligned multimodal windows.
2
+
3
+ ``WindowDataset`` is a plain sequence (``__len__`` and ``__getitem__``) of NumPy arrays, so it can be
4
+ used directly or wrapped by any data-loading framework. Each item holds every requested stream at its native
5
+ rate for the same time window, so EEG (125 Hz), motion (50 Hz) and PPG (25 Hz) stay aligned without
6
+ resampling. A window is only valid for a stream when it lies inside one uninterrupted segment with no
7
+ gap and no duplicated samples.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from dataclasses import dataclass
13
+ from pathlib import Path
14
+ from typing import Callable, Iterable, Sequence
15
+
16
+ import numpy as np
17
+
18
+ from .session import SIGNALS, Session, load_session
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class WindowRef:
23
+ session: int
24
+ t0: float
25
+
26
+
27
+ class _StreamIndex:
28
+ """Per-stream arrays and a fast validity check for windows."""
29
+
30
+ def __init__(self, session: Session, stream: str):
31
+ rate, _ = SIGNALS[stream]
32
+ self.rate = rate
33
+ self.t, self.values, self.segment = session.signal(stream)
34
+ self.values = self.values.astype(np.float32)
35
+
36
+ def take(self, t0: float, n: int) -> np.ndarray | None:
37
+ i = int(np.searchsorted(self.t, t0))
38
+ j = i + n
39
+ if j > len(self.t):
40
+ return None
41
+ tw = self.t[i:j]
42
+ period = 1.0 / self.rate
43
+ if tw[0] - t0 > 1.5 * period: # window starts in a gap
44
+ return None
45
+ if self.segment[i] != self.segment[j - 1]:
46
+ return None
47
+ if np.any(np.diff(tw) > 1.5 * period):
48
+ return None
49
+ return self.values[:, i:j]
50
+
51
+
52
+ class WindowDataset:
53
+ """Sliding windows over one or more sessions.
54
+
55
+ Parameters
56
+ ----------
57
+ sessions:
58
+ Session folders or loaded :class:`Session` objects.
59
+ window_s, hop_s:
60
+ Window length and step in seconds.
61
+ streams:
62
+ Streams to return (any of ``eeg``, ``accel``, ``gyro``, ``ppg``).
63
+ require:
64
+ Streams that must be valid for a window to be included; other streams are returned
65
+ zero-filled with ``mask[stream] = False`` when unavailable.
66
+ min_contact:
67
+ If set, skip windows where the band's contact state (0–3) drops below this value.
68
+ transform:
69
+ Optional callable applied to each item dict.
70
+ """
71
+
72
+ def __init__(
73
+ self,
74
+ sessions: Iterable[str | Path | Session],
75
+ window_s: float = 10.0,
76
+ hop_s: float = 5.0,
77
+ streams: Sequence[str] = ("eeg", "accel", "gyro", "ppg"),
78
+ require: Sequence[str] = ("eeg",),
79
+ min_contact: int | None = None,
80
+ transform: Callable[[dict], dict] | None = None,
81
+ ):
82
+ unknown = set(streams) - set(SIGNALS)
83
+ if unknown:
84
+ raise ValueError(f"unknown streams {sorted(unknown)}")
85
+ self.sessions = [s if isinstance(s, Session) else load_session(s) for s in sessions]
86
+ self.window_s, self.hop_s = float(window_s), float(hop_s)
87
+ self.streams, self.require = tuple(streams), tuple(require)
88
+ self.min_contact, self.transform = min_contact, transform
89
+ self.lengths = {s: int(round(self.window_s * SIGNALS[s][0])) for s in self.streams}
90
+ self._index = [{s: _StreamIndex(sess, s) for s in self.streams if s in sess} for sess in self.sessions]
91
+ self.windows: list[WindowRef] = []
92
+ self.skipped = 0
93
+ for k, sess in enumerate(self.sessions):
94
+ idx = self._index[k]
95
+ if any(r not in idx for r in self.require):
96
+ continue
97
+ start, end = sess.span(self.require)
98
+ if not np.isfinite(start):
99
+ continue
100
+ contact = sess.tables.get("contact")
101
+ # The span ends at the last sample's time, one period before the data ends.
102
+ last_period = 1.0 / min(SIGNALS[r][0] for r in self.require)
103
+ for t0 in np.arange(start, end - self.window_s + last_period + 1e-4, self.hop_s):
104
+ ok = all(idx[r].take(t0, self.lengths[r]) is not None for r in self.require)
105
+ if ok and self.min_contact is not None and contact is not None and len(contact):
106
+ c = contact[(contact["t"] >= t0) & (contact["t"] < t0 + self.window_s)]["state"]
107
+ ok = len(c) > 0 and int(c.min()) >= self.min_contact
108
+ if ok:
109
+ self.windows.append(WindowRef(k, float(t0)))
110
+ else:
111
+ self.skipped += 1
112
+
113
+ def __len__(self) -> int:
114
+ return len(self.windows)
115
+
116
+ def __getitem__(self, i: int) -> dict:
117
+ ref = self.windows[i]
118
+ idx = self._index[ref.session]
119
+ item: dict = {"session": self.sessions[ref.session].name, "t0": ref.t0, "mask": {}}
120
+ for s in self.streams:
121
+ channels = len(SIGNALS[s][1])
122
+ x = idx[s].take(ref.t0, self.lengths[s]) if s in idx else None
123
+ item["mask"][s] = x is not None
124
+ item[s] = x if x is not None else np.zeros((channels, self.lengths[s]), dtype=np.float32)
125
+ return self.transform(item) if self.transform else item
@@ -0,0 +1,89 @@
1
+ """Loading openfrenz session datasets (folders written by `frenz export`)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from dataclasses import dataclass, field
7
+ from pathlib import Path
8
+
9
+ import numpy as np
10
+ import pandas as pd
11
+
12
+ FORMAT = "openfrenz-session/1"
13
+
14
+ #: Stream name -> (nominal rate in Hz, signal columns).
15
+ SIGNALS: dict[str, tuple[float, tuple[str, ...]]] = {
16
+ "eeg": (125.0, ("forehead_right", "comb_left", "forehead_left", "comb_right")),
17
+ "accel": (50.0, ("x", "y", "z")),
18
+ "gyro": (50.0, ("x", "y", "z")),
19
+ # tag1 is always zero on verified hardware.
20
+ "ppg": (25.0, ("tag2", "tag3")),
21
+ }
22
+
23
+
24
+ @dataclass
25
+ class Session:
26
+ """One recording session: metadata plus one DataFrame per stream (sorted by time ``t``)."""
27
+
28
+ path: Path
29
+ meta: dict
30
+ tables: dict[str, pd.DataFrame] = field(default_factory=dict)
31
+
32
+ @property
33
+ def name(self) -> str:
34
+ return self.path.name
35
+
36
+ def __getitem__(self, stream: str) -> pd.DataFrame:
37
+ return self.tables[stream]
38
+
39
+ def __contains__(self, stream: str) -> bool:
40
+ return stream in self.tables
41
+
42
+ def span(self, streams=("eeg",)) -> tuple[float, float]:
43
+ """Time range (Unix s) covered by all of ``streams``."""
44
+ present = [self.tables[s] for s in streams if s in self.tables and len(self.tables[s])]
45
+ if not present:
46
+ return (np.nan, np.nan)
47
+ return (max(df["t"].iloc[0] for df in present), min(df["t"].iloc[-1] for df in present))
48
+
49
+ def slice(self, t0: float, t1: float) -> dict[str, pd.DataFrame]:
50
+ """All streams restricted to ``t0 <= t < t1``."""
51
+ out = {}
52
+ for name, df in self.tables.items():
53
+ t = df["t"].to_numpy()
54
+ out[name] = df.iloc[np.searchsorted(t, t0) : np.searchsorted(t, t1)]
55
+ return out
56
+
57
+ def signal(self, stream: str, include_duplicates: bool = False) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
58
+ """``(t, values[channels, n], segment)`` for a sampled stream. Rows flagged ``duplicate``
59
+ (a repeated block) or ``held`` (frozen sensor output) are removed unless asked for."""
60
+ df = self.tables[stream]
61
+ if not include_duplicates:
62
+ bad = np.zeros(len(df), dtype=bool)
63
+ for flag in ("duplicate", "held"):
64
+ if flag in df:
65
+ bad |= df[flag].to_numpy()
66
+ df = df[~bad]
67
+ cols = SIGNALS[stream][1]
68
+ seg = df["segment"].to_numpy() if "segment" in df else np.zeros(len(df), dtype=np.int64)
69
+ return df["t"].to_numpy(), df[list(cols)].to_numpy(dtype=np.float64).T, seg
70
+
71
+
72
+ def load_session(path: str | Path) -> Session:
73
+ """Load a session folder written by ``frenz export``."""
74
+ path = Path(path)
75
+ meta = json.loads((path / "session.json").read_text())
76
+ if meta.get("format") != FORMAT:
77
+ raise ValueError(f"{path}: unsupported format {meta.get('format')!r}")
78
+ tables = {}
79
+ for name, info in meta.get("tables", {}).items():
80
+ df = pd.read_parquet(path / info["file"])
81
+ if "t" in df and not df["t"].is_monotonic_increasing:
82
+ df = df.sort_values("t", kind="stable").reset_index(drop=True)
83
+ tables[name] = df
84
+ return Session(path=path, meta=meta, tables=tables)
85
+
86
+
87
+ def find_sessions(root: str | Path) -> list[Path]:
88
+ """All session folders (containing ``session.json``) under ``root``."""
89
+ return sorted(p.parent for p in Path(root).rglob("session.json"))
@@ -0,0 +1,19 @@
1
+ [project]
2
+ name = "openfrenz"
3
+ version = "0.2.1"
4
+ description = "Load openfrenz session datasets (FRENZ brainband) and build aligned fixed-length windows for analysis"
5
+ readme = "README.md"
6
+ license = "MIT"
7
+ requires-python = ">=3.10"
8
+ dependencies = ["numpy>=1.26", "pandas>=2.0", "pyarrow>=15"]
9
+
10
+ [project.urls]
11
+ Repository = "https://github.com/ljchang/openfrenz"
12
+ Documentation = "https://github.com/ljchang/openfrenz/blob/main/docs/CLI.md"
13
+
14
+ [project.optional-dependencies]
15
+ test = ["pytest>=8"]
16
+
17
+ [build-system]
18
+ requires = ["hatchling"]
19
+ build-backend = "hatchling.build"
@@ -0,0 +1,83 @@
1
+ """Tests on synthetic sessions, plus real exported sessions when OPENFRENZ_SESSIONS is set."""
2
+
3
+ import json
4
+ import os
5
+ from pathlib import Path
6
+
7
+ import numpy as np
8
+ import pandas as pd
9
+ import pytest
10
+
11
+ from openfrenz import WindowDataset, find_sessions, load_session
12
+
13
+
14
+ def write_session(root: Path, seconds=30.0, gap=None, duplicate_block=None) -> Path:
15
+ """A synthetic session: EEG 125 Hz, accel 50 Hz; optional gap (t0, t1) and duplicated block."""
16
+ root.mkdir(parents=True)
17
+ tables = {}
18
+ for name, rate, cols in [("eeg", 125.0, ["forehead_right", "comb_left", "forehead_left", "comb_right"]),
19
+ ("accel", 50.0, ["x", "y", "z"])]:
20
+ n = int(seconds * rate)
21
+ t = 1_800_000_000.0 + np.arange(n) / rate
22
+ segment = np.zeros(n, dtype=np.int64)
23
+ keep = np.ones(n, dtype=bool)
24
+ if gap:
25
+ keep &= ~((t >= gap[0] + t[0]) & (t < gap[1] + t[0]))
26
+ segment[t >= gap[1] + t[0]] = 1
27
+ df = pd.DataFrame({"t": t[keep], "segment": segment[keep]})
28
+ for k, c in enumerate(cols):
29
+ df[c] = np.sin(2 * np.pi * (k + 1) * df["t"].to_numpy()).astype(np.int32 if name == "eeg" else np.int16)
30
+ df["duplicate"] = False
31
+ if duplicate_block and name == "eeg":
32
+ a, b = duplicate_block
33
+ df.loc[(df["t"] >= t[0] + a) & (df["t"] < t[0] + b), "duplicate"] = True
34
+ df.to_parquet(root / f"{name}.parquet")
35
+ tables[name] = {"file": f"{name}.parquet", "rows": len(df)}
36
+ (root / "session.json").write_text(json.dumps({"format": "openfrenz-session/1", "tables": tables}))
37
+ return root
38
+
39
+
40
+ def test_windows_shapes_and_alignment(tmp_path):
41
+ s = write_session(tmp_path / "a", seconds=30)
42
+ ds = WindowDataset([s], window_s=10, hop_s=5, streams=("eeg", "accel"))
43
+ assert len(ds) == 5 # starts at 0, 5, 10, 15, 20 s
44
+ item = ds[0]
45
+ assert item["eeg"].shape == (4, 1250) and item["accel"].shape == (3, 500)
46
+ assert item["eeg"].dtype == np.float32
47
+ assert item["mask"] == {"eeg": True, "accel": True}
48
+
49
+
50
+ def test_gaps_and_duplicates_exclude_windows(tmp_path):
51
+ s = write_session(tmp_path / "b", seconds=40, gap=(12.0, 13.0))
52
+ ds = WindowDataset([s], window_s=10, hop_s=1)
53
+ assert all(not (ref.t0 - 1.8e9 < 13.0 and ref.t0 - 1.8e9 + 10 > 12.0) for ref in ds.windows)
54
+ assert ds.skipped > 0
55
+ d = write_session(tmp_path / "c", seconds=30, duplicate_block=(10.0, 11.0))
56
+ full = WindowDataset([write_session(tmp_path / "d", seconds=30)], window_s=10, hop_s=1)
57
+ dup = WindowDataset([d], window_s=10, hop_s=1)
58
+ assert len(dup) < len(full) # duplicated samples are removed, leaving a gap
59
+
60
+
61
+ def test_optional_streams_are_masked(tmp_path):
62
+ s = write_session(tmp_path / "e", seconds=20)
63
+ ds = WindowDataset([s], window_s=10, hop_s=10, streams=("eeg", "ppg"), require=("eeg",))
64
+ assert len(ds) >= 1
65
+ assert ds[0]["mask"]["ppg"] is False and ds[0]["ppg"].shape == (2, 250)
66
+
67
+
68
+ SESSIONS = os.environ.get("OPENFRENZ_SESSIONS")
69
+
70
+
71
+ @pytest.mark.skipif(not SESSIONS, reason="set OPENFRENZ_SESSIONS to a folder of exported sessions")
72
+ def test_real_sessions_load_and_align():
73
+ paths = find_sessions(SESSIONS)
74
+ assert paths
75
+ for p in paths:
76
+ s = load_session(p)
77
+ if "eeg" not in s:
78
+ continue
79
+ t, x, seg = s.signal("eeg")
80
+ assert np.all(np.diff(t)[np.diff(seg) == 0] > 0), f"{p.name}: time must increase within a segment"
81
+ ds = WindowDataset(paths, window_s=10, hop_s=5, streams=("eeg", "accel", "ppg"), require=("eeg",))
82
+ assert len(ds) > 0
83
+ assert ds[0]["eeg"].shape == (4, 1250)