plasma-app 2.1.3__tar.gz → 2.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {plasma_app-2.1.3 → plasma_app-2.2.2}/PKG-INFO +8 -1
- {plasma_app-2.1.3 → plasma_app-2.2.2}/README.md +7 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/__init__.py +1 -1
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/app_context.py +9 -4
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/desktop_shortcut.py +56 -5
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/extract/__main__.py +4 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/extract/options.py +18 -3
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/extract/pipeline.py +157 -45
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/formats/__init__.py +218 -142
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/extractor.py +3 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma_app.egg-info/PKG-INFO +8 -1
- {plasma_app-2.1.3 → plasma_app-2.2.2}/LICENSE +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/__main__.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/__pyinstaller/__init__.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/__pyinstaller/hook-plasma.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/api.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/build_info.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/config.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/ctl.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/__init__.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/bitalino.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/__init__.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/ble_scan.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/bleak_probe.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/bleak_reconnect_test.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/config.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/detect.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/device.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/extract/__init__.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/extract/clocksync.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/gyro_bias.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/nus_sim.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/nus_stream.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/__init__.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/_common.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/clocksync.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/control.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/downloader.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/imu.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/sqc.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/tools.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/uuid_tools.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/viewer.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/quaternion.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/records.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/signal_quality.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense_demo/__init__.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense_demo/config.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense_demo/device.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense_demo/fake_stream.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense_demo/faults.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/obs.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/pupil_labs.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/qb2.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/shimmer.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/template.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/integrated_panel.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/journal.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/lsl_recorder.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/lsl_session.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/lsl_util.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/plot_util.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/plugins.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/resources/icons/plasma.icns +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/resources/icons/plasma.ico +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/resources/icons/plasma.png +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/resources/icons/plasma.svg +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/status.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/xdf_writer.py +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma_app.egg-info/SOURCES.txt +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma_app.egg-info/dependency_links.txt +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma_app.egg-info/entry_points.txt +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma_app.egg-info/requires.txt +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma_app.egg-info/top_level.txt +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/pyproject.toml +0 -0
- {plasma_app-2.1.3 → plasma_app-2.2.2}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: plasma-app
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2.2
|
|
4
4
|
Summary: PLASMA — Platform for LSL-based Acquisition of Sensor Metrics and Analytics
|
|
5
5
|
License: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/YuyiChang/PLASMA
|
|
@@ -56,6 +56,13 @@ from faults.
|
|
|
56
56
|
|
|
57
57
|
## quickstart
|
|
58
58
|
|
|
59
|
+
**macOS: `brew install --cask yuyichang/plasma/plasma`** is the recommended
|
|
60
|
+
way to get PLASMA — no Python, no `liblsl`, no Gatekeeper dance. See
|
|
61
|
+
[Install & first launch](https://yuyichang.github.io/PLASMA/playbook/install/)
|
|
62
|
+
for that and every other install path (prebuilt binaries, PyPI, from source).
|
|
63
|
+
|
|
64
|
+
To run from source instead:
|
|
65
|
+
|
|
59
66
|
- `conda create -n plasma python=3.12`
|
|
60
67
|
- `conda activate plasma`
|
|
61
68
|
- `pip install -e ".[all]"` — or a lean subset, e.g. `".[msense]"` / `".[qb2,pupil]"`
|
|
@@ -8,6 +8,13 @@ from faults.
|
|
|
8
8
|
|
|
9
9
|
## quickstart
|
|
10
10
|
|
|
11
|
+
**macOS: `brew install --cask yuyichang/plasma/plasma`** is the recommended
|
|
12
|
+
way to get PLASMA — no Python, no `liblsl`, no Gatekeeper dance. See
|
|
13
|
+
[Install & first launch](https://yuyichang.github.io/PLASMA/playbook/install/)
|
|
14
|
+
for that and every other install path (prebuilt binaries, PyPI, from source).
|
|
15
|
+
|
|
16
|
+
To run from source instead:
|
|
17
|
+
|
|
11
18
|
- `conda create -n plasma python=3.12`
|
|
12
19
|
- `conda activate plasma`
|
|
13
20
|
- `pip install -e ".[all]"` — or a lean subset, e.g. `".[msense]"` / `".[qb2,pupil]"`
|
|
@@ -36,7 +36,7 @@ import os
|
|
|
36
36
|
import sys
|
|
37
37
|
from dataclasses import dataclass, replace
|
|
38
38
|
|
|
39
|
-
__all__ = ["AppContext", "app_context", "configure", "reset"]
|
|
39
|
+
__all__ = ["AppContext", "app_context", "configure", "reset", "user_data_dir"]
|
|
40
40
|
|
|
41
41
|
|
|
42
42
|
@dataclass(frozen=True)
|
|
@@ -71,8 +71,13 @@ _ctx: AppContext | None = None
|
|
|
71
71
|
_DEFAULT_APP_DIR = "PLASMA"
|
|
72
72
|
|
|
73
73
|
|
|
74
|
-
def
|
|
75
|
-
"""Per-user, per-OS application-data directory (stdlib, no platformdirs).
|
|
74
|
+
def user_data_dir(app: str = _DEFAULT_APP_DIR) -> str:
|
|
75
|
+
"""Per-user, per-OS application-data directory (stdlib, no platformdirs).
|
|
76
|
+
|
|
77
|
+
Public so other entry points (e.g. ``plasma.desktop_shortcut``) can target
|
|
78
|
+
the same directory a frozen build would resolve to, without themselves
|
|
79
|
+
running frozen.
|
|
80
|
+
"""
|
|
76
81
|
if sys.platform == "win32":
|
|
77
82
|
base = os.environ.get("LOCALAPPDATA") or os.path.join(
|
|
78
83
|
os.path.expanduser("~"), "AppData", "Local")
|
|
@@ -95,7 +100,7 @@ def _from_env() -> AppContext:
|
|
|
95
100
|
if env:
|
|
96
101
|
return AppContext(home=env)
|
|
97
102
|
if _is_frozen():
|
|
98
|
-
return AppContext(home=
|
|
103
|
+
return AppContext(home=user_data_dir())
|
|
99
104
|
return AppContext(home=os.getcwd())
|
|
100
105
|
|
|
101
106
|
|
|
@@ -14,9 +14,19 @@ Points the shortcut directly at *this* environment's ``plasma`` console-script
|
|
|
14
14
|
than re-deriving a `python -m ...` invocation — see the pyshortcuts prototype
|
|
15
15
|
notes in the packaging discussion for why.
|
|
16
16
|
|
|
17
|
+
The shortcut's working directory is pinned to the same per-OS app-data dir a
|
|
18
|
+
frozen PyInstaller build resolves to (``plasma.app_context.user_data_dir``),
|
|
19
|
+
not left to whatever the OS defaults a new shortcut to (``$HOME`` on
|
|
20
|
+
Windows/Linux; an unspecified Terminal default on macOS). Since
|
|
21
|
+
``plasma.app_context`` falls back to the working directory for a non-frozen
|
|
22
|
+
process, this keeps the Desktop-icon launch and a downloaded prebuilt binary
|
|
23
|
+
reading/writing config from the same place.
|
|
24
|
+
|
|
17
25
|
plasma-install-shortcut [--name NAME] [--no-terminal] [--no-startmenu]
|
|
18
26
|
|
|
19
|
-
Requires the ``desktop`` extra (``pip install plasma-app[desktop]``
|
|
27
|
+
Requires the ``desktop`` extra (``pip install "plasma-app[desktop]"`` —
|
|
28
|
+
quote it, or zsh's glob expansion on ``[...]`` will fail with "no matches
|
|
29
|
+
found").
|
|
20
30
|
"""
|
|
21
31
|
from __future__ import annotations
|
|
22
32
|
|
|
@@ -26,6 +36,8 @@ import shutil
|
|
|
26
36
|
import sys
|
|
27
37
|
from pathlib import Path
|
|
28
38
|
|
|
39
|
+
from plasma.app_context import user_data_dir
|
|
40
|
+
|
|
29
41
|
_ICON_DIR = Path(__file__).parent / "resources" / "icons"
|
|
30
42
|
|
|
31
43
|
|
|
@@ -60,6 +72,35 @@ def _plasma_executable() -> str:
|
|
|
60
72
|
)
|
|
61
73
|
|
|
62
74
|
|
|
75
|
+
def _target_home() -> str:
|
|
76
|
+
"""Where the shortcut should run PLASMA from — the same per-OS app-data
|
|
77
|
+
dir a frozen build resolves to (``$PLASMA_HOME`` still wins if set,
|
|
78
|
+
matching ``app_context``'s own precedence)."""
|
|
79
|
+
return os.environ.get("PLASMA_HOME") or user_data_dir()
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _launch_script(target_home: str) -> str:
|
|
83
|
+
"""Path handed to pyshortcuts as the thing to run.
|
|
84
|
+
|
|
85
|
+
On Windows/Linux, pyshortcuts writes ``target_home`` into the shortcut
|
|
86
|
+
itself (the ``.lnk`` ``WorkingDirectory`` / the ``.desktop`` ``Path=``
|
|
87
|
+
key), so the plain console-script is enough. On macOS, pyshortcuts'
|
|
88
|
+
Terminal/Automator launcher never applies ``working_dir`` at all — so we
|
|
89
|
+
wrap the console-script in a tiny shell script that ``cd``s first.
|
|
90
|
+
"""
|
|
91
|
+
exe = _plasma_executable()
|
|
92
|
+
if sys.platform != "darwin":
|
|
93
|
+
return exe
|
|
94
|
+
wrapper = Path(target_home) / "plasma-launch.sh"
|
|
95
|
+
wrapper.write_text(
|
|
96
|
+
"#!/bin/sh\n"
|
|
97
|
+
f'cd "{target_home}" || exit 1\n'
|
|
98
|
+
f'exec "{exe}"\n'
|
|
99
|
+
)
|
|
100
|
+
wrapper.chmod(0o755)
|
|
101
|
+
return str(wrapper)
|
|
102
|
+
|
|
103
|
+
|
|
63
104
|
def install_shortcut(name="PLASMA", terminal=True, startmenu=True, folder=None):
|
|
64
105
|
"""Create the Desktop (+ Start Menu, if requested) shortcut. Returns the
|
|
65
106
|
``pyshortcuts.Shortcut`` namedtuple describing what was written.
|
|
@@ -76,19 +117,23 @@ def install_shortcut(name="PLASMA", terminal=True, startmenu=True, folder=None):
|
|
|
76
117
|
except ImportError as e:
|
|
77
118
|
raise ImportError(
|
|
78
119
|
"pyshortcuts is required to create a desktop shortcut — install "
|
|
79
|
-
|
|
120
|
+
'it with `pip install "plasma-app[desktop]"`.'
|
|
80
121
|
) from e
|
|
81
122
|
|
|
123
|
+
target_home = _target_home()
|
|
124
|
+
os.makedirs(target_home, exist_ok=True)
|
|
125
|
+
|
|
82
126
|
return make_shortcut(
|
|
83
|
-
script=
|
|
127
|
+
script=_launch_script(target_home),
|
|
84
128
|
name=name,
|
|
85
129
|
description="PLASMA sensor acquisition dashboard",
|
|
86
130
|
icon=_icon_path(),
|
|
87
131
|
terminal=terminal,
|
|
88
132
|
desktop=True,
|
|
89
133
|
startmenu=startmenu, # pyshortcuts itself no-ops this on macOS
|
|
134
|
+
working_dir=target_home,
|
|
90
135
|
folder=folder,
|
|
91
|
-
noexe=True, # script is already a standalone console-script
|
|
136
|
+
noexe=True, # script is already a standalone console-script/wrapper
|
|
92
137
|
)
|
|
93
138
|
|
|
94
139
|
|
|
@@ -114,7 +159,13 @@ def main(argv=None):
|
|
|
114
159
|
startmenu=not args.no_startmenu,
|
|
115
160
|
)
|
|
116
161
|
print(f"Created shortcut: {scut.target}")
|
|
117
|
-
|
|
162
|
+
# NOTE: not scut.script/scut.full_script — pyshortcuts splits `script` on
|
|
163
|
+
# its first space to separate a script from trailing arguments, which
|
|
164
|
+
# mangles this when the path itself contains a space (e.g. macOS's
|
|
165
|
+
# "Application Support"). The shortcut it writes is still correct; only
|
|
166
|
+
# the returned namedtuple's script fields are truncated for display.
|
|
167
|
+
print(f" runs: {_plasma_executable()}")
|
|
168
|
+
print(f" in: {scut.working_dir}")
|
|
118
169
|
print(f" desktop: {scut.desktop_dir}")
|
|
119
170
|
if scut.startmenu_dir:
|
|
120
171
|
print(f" menu: {scut.startmenu_dir}")
|
|
@@ -39,6 +39,9 @@ def _add_extract_flags(p):
|
|
|
39
39
|
p.add_argument('--sniff_threshold', type=float, default=0.90)
|
|
40
40
|
p.add_argument('--dry_run', action='store_true', default=False)
|
|
41
41
|
p.add_argument('--strict_ppg', action='store_true', default=False)
|
|
42
|
+
p.add_argument('--include_cdct', action='store_true', default=False,
|
|
43
|
+
help="PPG-device output (ppg/ac legacy/v2) keeps CDCT/init_CDCT/Datetime; "
|
|
44
|
+
"off by default (start time is always in README.txt)")
|
|
42
45
|
p.add_argument('--note', default="", help="note recorded in the output README")
|
|
43
46
|
|
|
44
47
|
|
|
@@ -68,6 +71,7 @@ def main(argv=None):
|
|
|
68
71
|
p_batch.add_argument('--on_format_conflict', choices=CONFLICT_CHOICES, default='warn')
|
|
69
72
|
p_batch.add_argument('--sniff_threshold', type=float, default=0.90)
|
|
70
73
|
p_batch.add_argument('--strict_ppg', action='store_true', default=False)
|
|
74
|
+
p_batch.add_argument('--include_cdct', action='store_true', default=False)
|
|
71
75
|
p_batch.add_argument('--legacy_fs', action='store_true', default=False)
|
|
72
76
|
p_batch.add_argument('--dry_run', action='store_true', default=False)
|
|
73
77
|
p_batch.add_argument('--note', default="")
|
|
@@ -31,13 +31,13 @@ needed, and where the two disagree the contents win.
|
|
|
31
31
|
|---|---|
|
|
32
32
|
| `auto` | Detect from content, per file. Falls back to the device version if inconclusive. **Default.** |
|
|
33
33
|
| `version` | Follow `uuid.txt` only (v4.7.0+ → `v2`, otherwise `legacy`). The pre-1.6 behaviour. |
|
|
34
|
-
| `legacy` / `v2` / `packed16` / `
|
|
34
|
+
| `legacy` / `v2` / `packed16` / `v3` / `block_v2` | Force that layout. |
|
|
35
35
|
|
|
36
36
|
| Sensor | Layouts |
|
|
37
37
|
|---|---|
|
|
38
38
|
| PPG | `legacy` 24 B · `v2` 20 B · `packed16` 16 B (no version tie) |
|
|
39
|
-
| IMU | `legacy` 30 B record · `v2` 26 B record · `v3` self-describing 4 MiB chunk (no version tie)
|
|
40
|
-
| ECG | `
|
|
39
|
+
| IMU | `legacy` 30 B record (wristband) · `v2` 26 B record (wristband) · `v3` self-describing `ACF3` 4 MiB chunk (no version tie; MSense4ECG-Z5G4A chest device) — output columns for `v3` are locked to `SampleSequence`/`RtcTickEstBlock`/`RtcTickEst`/`AccX`/`AccY`/`AccZ` |
|
|
40
|
+
| ECG | `block_v2` self-describing `ECF2` 4 MiB chunk of 4096 B `ECB2` blocks (no version tie) — the only ECG layout; output columns are locked to `SampleIndex`/`RtcTick`/`ECG`/`ETAG`/`PTAG` |
|
|
41
41
|
|
|
42
42
|
**Cross-check against uuid.txt** reports when detection and the version file
|
|
43
43
|
disagree. It is off by default and does not override detection: use
|
|
@@ -46,6 +46,14 @@ disagree. It is off by default and does not override detection: use
|
|
|
46
46
|
**Strict record validation**: raise on a record that fails its integrity check
|
|
47
47
|
(packed16 reserved bits, ECG CRC, a partial trailing record) instead of dropping
|
|
48
48
|
it and reporting a count.
|
|
49
|
+
|
|
50
|
+
**Include CDCT/Datetime** (off by default): PPG/IMU `legacy`/`v2`/`packed16`
|
|
51
|
+
output no longer carries `CDCT`/`init_CDCT`/`Datetime` unless this is ticked —
|
|
52
|
+
`ecg:block_v2` and IMU `v3` never had these columns to begin with (see their
|
|
53
|
+
locked schemas above). Every PPG-device file's start time (UTC, from its
|
|
54
|
+
filename) is always recorded in `README.txt` regardless of this setting, so
|
|
55
|
+
turn it on only if you need the per-row columns themselves (e.g. to feed
|
|
56
|
+
`clocksync.py`, which is CSV/`Counter`+`CDCT`-based).
|
|
49
57
|
"""
|
|
50
58
|
|
|
51
59
|
|
|
@@ -71,6 +79,12 @@ class ExtractionOptions:
|
|
|
71
79
|
on_format_conflict: str = "warn"
|
|
72
80
|
strict_ppg: bool = False
|
|
73
81
|
force_new_format: bool = False
|
|
82
|
+
# Off by default: PPG-device output (ppg:*, ac:legacy/v2) no longer carries
|
|
83
|
+
# CDCT/init_CDCT/Datetime. ecg:block_v2 and ac:v3 never had these columns
|
|
84
|
+
# regardless of this setting (see formats.py's locked schemas). Each
|
|
85
|
+
# PPG-device file's start time is always written to README.txt instead —
|
|
86
|
+
# see DataExtractor.write_provenance.
|
|
87
|
+
include_cdct: bool = False
|
|
74
88
|
sniff_threshold: float = field(default=0.90, metadata={"panel": False})
|
|
75
89
|
dry_run: bool = field(default=False, metadata={"panel": False})
|
|
76
90
|
|
|
@@ -96,6 +110,7 @@ class ExtractionOptions:
|
|
|
96
110
|
on_format_conflict=args.on_format_conflict,
|
|
97
111
|
strict_ppg=args.strict_ppg,
|
|
98
112
|
force_new_format=args.force_new_format,
|
|
113
|
+
include_cdct=args.include_cdct,
|
|
99
114
|
sniff_threshold=args.sniff_threshold,
|
|
100
115
|
dry_run=args.dry_run,
|
|
101
116
|
)
|
|
@@ -177,6 +177,8 @@ class DataExtractor():
|
|
|
177
177
|
self.malformed = 0
|
|
178
178
|
self.dropped = 0
|
|
179
179
|
self.out_paths = []
|
|
180
|
+
self.anchor_block_summaries = [] # (basename, first, last, n_ok, n_bad) — README-only
|
|
181
|
+
self.ppg_file_start_times = [] # (basename, UTC datetime str) — README-only, PPG-device files only
|
|
180
182
|
|
|
181
183
|
self.encoding_alias = self.get_encoding_alias() if self.df is not None else {}
|
|
182
184
|
|
|
@@ -197,6 +199,7 @@ class DataExtractor():
|
|
|
197
199
|
file.write(f"Cross-check against uuid.txt = {options.validate_with_uuid}"
|
|
198
200
|
f" (on conflict: {options.on_format_conflict})\n")
|
|
199
201
|
file.write(f"Strict record validation = {options.strict_ppg}\n")
|
|
202
|
+
file.write(f"Include CDCT/init_CDCT/Datetime (PPG-device output) = {options.include_cdct}\n")
|
|
200
203
|
file.write(f"Detection threshold = {options.sniff_threshold}\n")
|
|
201
204
|
file.write("I m-sense with YAMS at https://github.com/SenSE-Lab-OSU/YAMS\n")
|
|
202
205
|
uuid_path = os.path.join(self.in_dir, "uuid.txt")
|
|
@@ -259,6 +262,22 @@ class DataExtractor():
|
|
|
259
262
|
if conflicts:
|
|
260
263
|
file.write(f"uuid.txt conflicts = {len(conflicts)} "
|
|
261
264
|
f"(content used unless on_format_conflict=trust_uuid)\n")
|
|
265
|
+
if self.anchor_block_summaries:
|
|
266
|
+
file.write("\n--- AC/ECG block metadata (per ACB1/ECB2 block header, "
|
|
267
|
+
'BLOCK_META_FMT = "<4sIII": magic, AnchorCounter '
|
|
268
|
+
"(reserved_timer_output / first_rtc_tick), "
|
|
269
|
+
"OrigCounter base (first_sample_sequence / first_sample_index), "
|
|
270
|
+
"block_crc32) ---\n")
|
|
271
|
+
for basename, first, last, n_ok, n_bad in self.anchor_block_summaries:
|
|
272
|
+
file.write(f"{basename}: AnchorCounter {first}..{last} "
|
|
273
|
+
f"({n_ok} block(s) ok, {n_bad} bad)\n")
|
|
274
|
+
if self.ppg_file_start_times:
|
|
275
|
+
# Recorded regardless of `include_cdct`: once that option is off
|
|
276
|
+
# (the default), this is the only place a PPG-device file's
|
|
277
|
+
# start time (its filename-derived t0) still shows up.
|
|
278
|
+
file.write("\n--- PPG-device file start times (UTC) ---\n")
|
|
279
|
+
for basename, dt in sorted(self.ppg_file_start_times):
|
|
280
|
+
file.write(f"{basename}: {dt}\n")
|
|
262
281
|
|
|
263
282
|
def resolve(self, full_path, sensor):
|
|
264
283
|
res = detect.resolve(
|
|
@@ -273,9 +292,23 @@ class DataExtractor():
|
|
|
273
292
|
|
|
274
293
|
def read_file(self, full_path, sensor):
|
|
275
294
|
res = self.resolve(full_path, sensor)
|
|
276
|
-
df, dt = formats.read_bin(full_path, res.spec, strict=self.strict
|
|
295
|
+
df, dt = formats.read_bin(full_path, res.spec, strict=self.strict,
|
|
296
|
+
include_cdct=self.options.include_cdct)
|
|
277
297
|
self.malformed += df.attrs.get('malformed_records', 0)
|
|
278
298
|
self.dropped += df.attrs.get('dropped_samples', 0)
|
|
299
|
+
if res.spec.key in ("ac:v3", "ecg:block_v2"):
|
|
300
|
+
self.anchor_block_summaries.append((
|
|
301
|
+
os.path.basename(full_path),
|
|
302
|
+
df.attrs.get("anchor_counter_first"),
|
|
303
|
+
df.attrs.get("anchor_counter_last"),
|
|
304
|
+
df.attrs.get("n_blocks_ok", 0),
|
|
305
|
+
df.attrs.get("n_blocks_bad", 0),
|
|
306
|
+
))
|
|
307
|
+
else:
|
|
308
|
+
# A flat PPG-device format (ppg:*, ac:legacy/v2) — the two
|
|
309
|
+
# container formats (ac:v3, ecg:block_v2) belong to the
|
|
310
|
+
# MSense4ECG chest device and are excluded here.
|
|
311
|
+
self.ppg_file_start_times.append((os.path.basename(full_path), dt))
|
|
279
312
|
return df, res
|
|
280
313
|
|
|
281
314
|
def extract_csv(self, search_prefix, file_name, id=-1):
|
|
@@ -326,7 +359,8 @@ class DataExtractor():
|
|
|
326
359
|
# several distinct recordings, so a legitimate gap *between*
|
|
327
360
|
# recordings is never mistaken for a dropped-sample run within one.
|
|
328
361
|
counter_validity_check(chunk, spec)
|
|
329
|
-
chunk = _finish_chunk(chunk, is_ac=is_ac, spec=spec
|
|
362
|
+
chunk = _finish_chunk(chunk, is_ac=is_ac, spec=spec,
|
|
363
|
+
include_cdct=self.options.include_cdct)
|
|
330
364
|
if whole_frame:
|
|
331
365
|
pieces.append(chunk)
|
|
332
366
|
else:
|
|
@@ -353,16 +387,21 @@ class DataExtractor():
|
|
|
353
387
|
one-chunk "session") and, for any session spanning more than one chunk,
|
|
354
388
|
restitch CDCT as one continuous clock anchored at that session's t0.
|
|
355
389
|
|
|
356
|
-
Container formats need no CDCT recompute
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
`
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
390
|
+
Container formats need no CDCT recompute — both are locked to
|
|
391
|
+
ECG_EXTRACTION_HANDOFF.md's schemas and have no CDCT column at all:
|
|
392
|
+
- `ac:v3` (ACF3) columns are `SampleSequence`, `RtcTickEstBlock`,
|
|
393
|
+
`RtcTickEst`, `AccX`, `AccY`, `AccZ`. Chunks are re-ordered by
|
|
394
|
+
`chunk_index`; a `first_sample_sequence` discontinuity at a chunk
|
|
395
|
+
boundary (firmware drop, or a missing chunk) is reported, and
|
|
396
|
+
`RtcTickEst` (the piecewise-linear per-sample tick ramp) is built
|
|
397
|
+
across the whole session's blocks — see `_stitch_ac_v3_chunks`.
|
|
398
|
+
- `ecg:block_v2` (ECF2) has no CDCT at all: its output is locked to
|
|
399
|
+
ECG_EXTRACTION_HANDOFF.md's schema (`SampleIndex`, `RtcTick`, `ECG`,
|
|
400
|
+
`ETAG`, `PTAG`). `SampleIndex` is the recording-local sample ordinal
|
|
401
|
+
that already advances across chunks, so plain concatenation is
|
|
402
|
+
continuous. ECF2 chunks are re-ordered by `chunk_index` and split by
|
|
403
|
+
`recording_id`, and a gap in `chunk_index` or `SampleIndex` is
|
|
404
|
+
reported (not silently joined).
|
|
366
405
|
"""
|
|
367
406
|
session_dfs, spec = self._collect_session_frames(path, prefix)
|
|
368
407
|
if not session_dfs:
|
|
@@ -405,7 +444,10 @@ class DataExtractor():
|
|
|
405
444
|
session_dfs.append(dfs[0])
|
|
406
445
|
else:
|
|
407
446
|
combined = pd.concat(dfs, ignore_index=True)
|
|
408
|
-
|
|
447
|
+
# flat per-record formats only, and only when CDCT is wanted —
|
|
448
|
+
# `read_bin` already omitted it per-chunk when include_cdct is
|
|
449
|
+
# off (the default), so re-stitching it here would put it back.
|
|
450
|
+
if spec.read_file is None and self.options.include_cdct:
|
|
409
451
|
combined = formats.recompute_cdct(combined, spec, t0)
|
|
410
452
|
session_dfs.append(combined)
|
|
411
453
|
|
|
@@ -442,11 +484,44 @@ def gather_files_by_prefix(prefix: str, path):
|
|
|
442
484
|
return all_files
|
|
443
485
|
|
|
444
486
|
|
|
487
|
+
def _piecewise_linear_ac_tick(block_ticks, block_n_samples):
|
|
488
|
+
"""Per-sample `RtcTickEst`: within each block, ramp linearly from that
|
|
489
|
+
block's own `RtcTickEstBlock` anchor toward the *next* block's anchor,
|
|
490
|
+
dividing the actual elapsed ticks by that block's actual sample count (not
|
|
491
|
+
the nominal 680) — so real per-block rate variation is preserved, and the
|
|
492
|
+
ramp is exactly continuous at block boundaries (block i+1 starts exactly
|
|
493
|
+
at its own anchor). `block_ticks`/`block_n_samples` must already span the
|
|
494
|
+
whole session (every chunk's blocks, in `chunk_index` order — see
|
|
495
|
+
`_stitch_ac_v3_chunks`), since the last block of one chunk ramps toward
|
|
496
|
+
the first block of the next. The session's last block has no next anchor
|
|
497
|
+
to ramp toward; it extrapolates using the previous block's local rate.
|
|
498
|
+
Ported from ECG_EXTRACTION_HANDOFF.md's `extract_ecg_ac_v3.py`.
|
|
499
|
+
"""
|
|
500
|
+
n_blocks = len(block_ticks)
|
|
501
|
+
segments = []
|
|
502
|
+
prev_rate = 0.0
|
|
503
|
+
for i in range(n_blocks):
|
|
504
|
+
n = block_n_samples[i]
|
|
505
|
+
if i < n_blocks - 1:
|
|
506
|
+
rate = (block_ticks[i + 1] - block_ticks[i]) / n
|
|
507
|
+
else:
|
|
508
|
+
rate = prev_rate
|
|
509
|
+
segments.append(block_ticks[i] + rate * np.arange(n, dtype=np.float64))
|
|
510
|
+
prev_rate = rate
|
|
511
|
+
return np.concatenate(segments) if segments else np.array([], dtype=np.float64)
|
|
512
|
+
|
|
513
|
+
|
|
445
514
|
def _stitch_ecb2_chunks(dfs):
|
|
446
515
|
"""Order ECF2 chunk DataFrames by `recording_id` then `chunk_index`,
|
|
447
|
-
report any `chunk_index` or `
|
|
448
|
-
concatenated DataFrame per recording
|
|
449
|
-
|
|
516
|
+
report any `chunk_index` or `SampleIndex` discontinuity, and return one
|
|
517
|
+
concatenated DataFrame per recording.
|
|
518
|
+
|
|
519
|
+
Unlike `_stitch_ac_v3_chunks`, no global `Counter` offset is added:
|
|
520
|
+
`SampleIndex` is already the recording-local sample ordinal, continuous by
|
|
521
|
+
construction across every chunk of one `recording_id` (see
|
|
522
|
+
`formats._read_ecf2`), so plain concatenation is the whole job — the
|
|
523
|
+
output column set stays locked to ECG_EXTRACTION_HANDOFF.md's schema.
|
|
524
|
+
"""
|
|
450
525
|
by_recording = {}
|
|
451
526
|
for df in dfs:
|
|
452
527
|
by_recording.setdefault(df.attrs.get("recording_id"), []).append(df)
|
|
@@ -471,13 +546,23 @@ def _stitch_ecb2_chunks(dfs):
|
|
|
471
546
|
|
|
472
547
|
|
|
473
548
|
def _stitch_ac_v3_chunks(dfs):
|
|
474
|
-
"""Order ACF3 chunk DataFrames by `chunk_index
|
|
549
|
+
"""Order ACF3 chunk DataFrames by `chunk_index`, report a
|
|
475
550
|
`first_sample_sequence` discontinuity at a chunk boundary — a firmware drop
|
|
476
551
|
("a larger difference records the number of missing samples", per the
|
|
477
|
-
format doc) or a missing chunk
|
|
478
|
-
|
|
552
|
+
format doc) or a missing chunk — and return one concatenated DataFrame for
|
|
553
|
+
the session with `RtcTickEst` added.
|
|
554
|
+
|
|
555
|
+
`RtcTickEst` (the piecewise-linear per-sample tick ramp) is computed here,
|
|
556
|
+
not per-chunk in `formats._read_ac_v3`, because it can span a chunk
|
|
557
|
+
boundary: the last block of one chunk ramps toward the first block of the
|
|
558
|
+
next, so the full session's `(block_tick, n_samples)` list — gathered from
|
|
559
|
+
every chunk's `block_ticks`/`block_n_samples` attrs, in `chunk_index`
|
|
560
|
+
order — must be known first. Output columns otherwise stay locked to
|
|
561
|
+
ECG_EXTRACTION_HANDOFF.md's schema: no global `Counter` offset is added.
|
|
562
|
+
"""
|
|
479
563
|
group = sorted(dfs, key=lambda d: d.attrs.get("chunk_index", 0))
|
|
480
564
|
prev = None
|
|
565
|
+
session_block_ticks, session_block_n = [], []
|
|
481
566
|
for d in group:
|
|
482
567
|
ci = d.attrs.get("chunk_index", 0)
|
|
483
568
|
if ci == 0 and d.attrs.get("first_seq") not in (None, 0):
|
|
@@ -494,7 +579,11 @@ def _stitch_ac_v3_chunks(dfs):
|
|
|
494
579
|
print(f"AC session: {gap} sample(s) dropped at the chunk "
|
|
495
580
|
f"{ci} boundary")
|
|
496
581
|
prev = d
|
|
497
|
-
|
|
582
|
+
session_block_ticks.extend(d.attrs.get("block_ticks", []))
|
|
583
|
+
session_block_n.extend(d.attrs.get("block_n_samples", []))
|
|
584
|
+
combined = pd.concat(group, ignore_index=True)
|
|
585
|
+
combined["RtcTickEst"] = _piecewise_linear_ac_tick(session_block_ticks, session_block_n)
|
|
586
|
+
return combined
|
|
498
587
|
|
|
499
588
|
|
|
500
589
|
def counter_validity_check(df: pd.DataFrame, spec=None):
|
|
@@ -506,8 +595,19 @@ def counter_validity_check(df: pd.DataFrame, spec=None):
|
|
|
506
595
|
if spec is None:
|
|
507
596
|
print("pass counter check: N/A (no format resolved)")
|
|
508
597
|
return
|
|
509
|
-
#
|
|
510
|
-
|
|
598
|
+
# Flat ppg/ac layouts (legacy/v2/packed16) use 'Counter'. The two
|
|
599
|
+
# container formats are locked to ECG_EXTRACTION_HANDOFF.md's schema and
|
|
600
|
+
# have their own analogous column instead: ecg:block_v2's 'SampleIndex'
|
|
601
|
+
# and ac:v3's 'SampleSequence'. Absent all three the last column is not
|
|
602
|
+
# the counter either.
|
|
603
|
+
if 'Counter' in df.columns:
|
|
604
|
+
counter_columns = df[['Counter']]
|
|
605
|
+
elif 'SampleIndex' in df.columns:
|
|
606
|
+
counter_columns = df[['SampleIndex']]
|
|
607
|
+
elif 'SampleSequence' in df.columns:
|
|
608
|
+
counter_columns = df[['SampleSequence']]
|
|
609
|
+
else:
|
|
610
|
+
counter_columns = df.iloc[:, -1:]
|
|
511
611
|
counter_arr = numpy.array(counter_columns).flatten()
|
|
512
612
|
diff_arr = numpy.diff(counter_arr)
|
|
513
613
|
step = spec.tick_step
|
|
@@ -520,12 +620,12 @@ def counter_validity_check(df: pd.DataFrame, spec=None):
|
|
|
520
620
|
print("and number of non matching samples: " + str(numpy.count_nonzero(check_array == 0)))
|
|
521
621
|
|
|
522
622
|
|
|
523
|
-
def _finish_chunk(data_set, *, is_ac, spec):
|
|
524
|
-
"""Add the human-readable Datetime column
|
|
525
|
-
g — replacing a Python `datetime.fromtimestamp()` +
|
|
526
|
-
row, the dominant cost of the old single-shot
|
|
527
|
-
interpreted calls for a long high-rate
|
|
528
|
-
was blamed for.
|
|
623
|
+
def _finish_chunk(data_set, *, is_ac, spec, include_cdct):
|
|
624
|
+
"""Add the human-readable Datetime column (when `include_cdct`) and (for
|
|
625
|
+
AC) convert counts to g — replacing a Python `datetime.fromtimestamp()` +
|
|
626
|
+
`.strftime()` call per row, the dominant cost of the old single-shot
|
|
627
|
+
extraction (millions of interpreted calls for a long high-rate
|
|
628
|
+
recording), not the CSV write it was blamed for.
|
|
529
629
|
|
|
530
630
|
Deliberately NOT `pd.to_datetime(...).dt.strftime(...)`: benchmarked at
|
|
531
631
|
~2.9s for 2M rows vs. ~2.9s for the original per-row loop — pandas'
|
|
@@ -536,16 +636,31 @@ def _finish_chunk(data_set, *, is_ac, spec):
|
|
|
536
636
|
(`numpy.datetime_as_string`), then two vectorized character replaces to
|
|
537
637
|
turn ISO-8601 into this project's on-disk format — ~0.6s for the same 2M
|
|
538
638
|
rows, ~5x the original loop, verified to produce byte-identical strings.
|
|
639
|
+
|
|
640
|
+
ecg:block_v2 and ac:v3 output have no `CDCT` column at all — both are
|
|
641
|
+
locked to ECG_EXTRACTION_HANDOFF.md's schemas (`SampleIndex`/`RtcTick`/
|
|
642
|
+
`ECG`/`ETAG`/`PTAG` and `SampleSequence`/`RtcTickEstBlock`/`RtcTickEst`/
|
|
643
|
+
`AccX`/`AccY`/`AccZ` respectively — `AccX/Y/Z` are already in g, converted
|
|
644
|
+
in `formats._read_ac_v3`), so this is a no-op for both regardless of
|
|
645
|
+
`include_cdct` rather than fabricating a Datetime column or re-converting
|
|
646
|
+
units. For the flat ppg/ac formats, `include_cdct=False` (the default)
|
|
647
|
+
means `read_bin` already omitted `CDCT`/`init_CDCT` (see
|
|
648
|
+
`ExtractionOptions.include_cdct`), so this skips Datetime too — unit
|
|
649
|
+
conversion for AC still runs either way, since it has nothing to do with
|
|
650
|
+
CDCT.
|
|
539
651
|
"""
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
652
|
+
if spec is not None and spec.key in ("ecg:block_v2", "ac:v3"):
|
|
653
|
+
return data_set
|
|
654
|
+
if include_cdct:
|
|
655
|
+
try:
|
|
656
|
+
secs = data_set['CDCT'].to_numpy()
|
|
657
|
+
if not np.isfinite(secs).all():
|
|
658
|
+
raise ValueError("non-finite CDCT value(s) — can't convert to a timestamp")
|
|
659
|
+
iso = np.datetime_as_string(secs.astype(np.int64).astype('datetime64[s]'), unit='s')
|
|
660
|
+
data_set['Datetime'] = np.char.replace(np.char.replace(iso, '-', '/'), 'T', ' ')
|
|
661
|
+
except Exception as e:
|
|
662
|
+
print(str(e))
|
|
663
|
+
data_set['Datetime'] = -1
|
|
549
664
|
if is_ac:
|
|
550
665
|
print("perform unit conversion for IMU")
|
|
551
666
|
data_set = unit_conversion_ac(data_set, spec)
|
|
@@ -553,15 +668,12 @@ def _finish_chunk(data_set, *, is_ac, spec):
|
|
|
553
668
|
|
|
554
669
|
|
|
555
670
|
def unit_conversion_ac(data_set, spec=None):
|
|
556
|
-
"""Raw counts -> g
|
|
557
|
-
|
|
671
|
+
"""Raw counts -> g, for the legacy/v2 (wristband) IMU layouts — kept
|
|
672
|
+
unchanged. ac:v3's own scale is applied in `formats._read_ac_v3` instead
|
|
673
|
+
(its output is already in g), since `_finish_chunk` skips this call for it.
|
|
558
674
|
"""
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
data_set[c] = data_set[c] / formats.AC_V3_COUNTS_PER_G
|
|
562
|
-
else:
|
|
563
|
-
for c in ['AccX', 'AccY', 'AccZ']:
|
|
564
|
-
data_set[c] = data_set[c] / (2**16 - 1) * 8
|
|
675
|
+
for c in ['AccX', 'AccY', 'AccZ']:
|
|
676
|
+
data_set[c] = data_set[c] / (2**16 - 1) * 8
|
|
565
677
|
return data_set
|
|
566
678
|
|
|
567
679
|
|