plasma-app 2.1.3__tar.gz → 2.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. {plasma_app-2.1.3 → plasma_app-2.2.2}/PKG-INFO +8 -1
  2. {plasma_app-2.1.3 → plasma_app-2.2.2}/README.md +7 -0
  3. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/__init__.py +1 -1
  4. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/app_context.py +9 -4
  5. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/desktop_shortcut.py +56 -5
  6. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/extract/__main__.py +4 -0
  7. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/extract/options.py +18 -3
  8. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/extract/pipeline.py +157 -45
  9. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/formats/__init__.py +218 -142
  10. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/extractor.py +3 -0
  11. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma_app.egg-info/PKG-INFO +8 -1
  12. {plasma_app-2.1.3 → plasma_app-2.2.2}/LICENSE +0 -0
  13. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/__main__.py +0 -0
  14. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/__pyinstaller/__init__.py +0 -0
  15. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/__pyinstaller/hook-plasma.py +0 -0
  16. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/api.py +0 -0
  17. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/build_info.py +0 -0
  18. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/config.py +0 -0
  19. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/ctl.py +0 -0
  20. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/__init__.py +0 -0
  21. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/bitalino.py +0 -0
  22. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/__init__.py +0 -0
  23. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/ble_scan.py +0 -0
  24. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/bleak_probe.py +0 -0
  25. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/bleak_reconnect_test.py +0 -0
  26. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/config.py +0 -0
  27. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/detect.py +0 -0
  28. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/device.py +0 -0
  29. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/extract/__init__.py +0 -0
  30. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/extract/clocksync.py +0 -0
  31. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/gyro_bias.py +0 -0
  32. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/nus_sim.py +0 -0
  33. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/nus_stream.py +0 -0
  34. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/__init__.py +0 -0
  35. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/_common.py +0 -0
  36. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/clocksync.py +0 -0
  37. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/control.py +0 -0
  38. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/downloader.py +0 -0
  39. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/imu.py +0 -0
  40. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/sqc.py +0 -0
  41. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/tools.py +0 -0
  42. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/uuid_tools.py +0 -0
  43. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/panels/viewer.py +0 -0
  44. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/quaternion.py +0 -0
  45. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/records.py +0 -0
  46. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense/signal_quality.py +0 -0
  47. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense_demo/__init__.py +0 -0
  48. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense_demo/config.py +0 -0
  49. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense_demo/device.py +0 -0
  50. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense_demo/fake_stream.py +0 -0
  51. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/msense_demo/faults.py +0 -0
  52. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/obs.py +0 -0
  53. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/pupil_labs.py +0 -0
  54. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/qb2.py +0 -0
  55. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/shimmer.py +0 -0
  56. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/devices/template.py +0 -0
  57. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/integrated_panel.py +0 -0
  58. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/journal.py +0 -0
  59. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/lsl_recorder.py +0 -0
  60. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/lsl_session.py +0 -0
  61. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/lsl_util.py +0 -0
  62. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/plot_util.py +0 -0
  63. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/plugins.py +0 -0
  64. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/resources/icons/plasma.icns +0 -0
  65. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/resources/icons/plasma.ico +0 -0
  66. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/resources/icons/plasma.png +0 -0
  67. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/resources/icons/plasma.svg +0 -0
  68. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/status.py +0 -0
  69. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma/xdf_writer.py +0 -0
  70. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma_app.egg-info/SOURCES.txt +0 -0
  71. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma_app.egg-info/dependency_links.txt +0 -0
  72. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma_app.egg-info/entry_points.txt +0 -0
  73. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma_app.egg-info/requires.txt +0 -0
  74. {plasma_app-2.1.3 → plasma_app-2.2.2}/plasma_app.egg-info/top_level.txt +0 -0
  75. {plasma_app-2.1.3 → plasma_app-2.2.2}/pyproject.toml +0 -0
  76. {plasma_app-2.1.3 → plasma_app-2.2.2}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: plasma-app
3
- Version: 2.1.3
3
+ Version: 2.2.2
4
4
  Summary: PLASMA — Platform for LSL-based Acquisition of Sensor Metrics and Analytics
5
5
  License: MIT
6
6
  Project-URL: Homepage, https://github.com/YuyiChang/PLASMA
@@ -56,6 +56,13 @@ from faults.
56
56
 
57
57
  ## quickstart
58
58
 
59
+ **macOS: `brew install --cask yuyichang/plasma/plasma`** is the recommended
60
+ way to get PLASMA — no Python, no `liblsl`, no Gatekeeper dance. See
61
+ [Install & first launch](https://yuyichang.github.io/PLASMA/playbook/install/)
62
+ for that and every other install path (prebuilt binaries, PyPI, from source).
63
+
64
+ To run from source instead:
65
+
59
66
  - `conda create -n plasma python=3.12`
60
67
  - `conda activate plasma`
61
68
  - `pip install -e ".[all]"` — or a lean subset, e.g. `".[msense]"` / `".[qb2,pupil]"`
@@ -8,6 +8,13 @@ from faults.
8
8
 
9
9
  ## quickstart
10
10
 
11
+ **macOS: `brew install --cask yuyichang/plasma/plasma`** is the recommended
12
+ way to get PLASMA — no Python, no `liblsl`, no Gatekeeper dance. See
13
+ [Install & first launch](https://yuyichang.github.io/PLASMA/playbook/install/)
14
+ for that and every other install path (prebuilt binaries, PyPI, from source).
15
+
16
+ To run from source instead:
17
+
11
18
  - `conda create -n plasma python=3.12`
12
19
  - `conda activate plasma`
13
20
  - `pip install -e ".[all]"` — or a lean subset, e.g. `".[msense]"` / `".[qb2,pupil]"`
@@ -1,4 +1,4 @@
1
- __version__ = "2.1.3"
1
+ __version__ = "2.2.2"
2
2
 
3
3
  # Short git commit hash baked in by CI (.github/workflows/build.yml) just
4
4
  # before a release build is frozen with PyInstaller — a frozen bundle ships no
@@ -36,7 +36,7 @@ import os
36
36
  import sys
37
37
  from dataclasses import dataclass, replace
38
38
 
39
- __all__ = ["AppContext", "app_context", "configure", "reset"]
39
+ __all__ = ["AppContext", "app_context", "configure", "reset", "user_data_dir"]
40
40
 
41
41
 
42
42
  @dataclass(frozen=True)
@@ -71,8 +71,13 @@ _ctx: AppContext | None = None
71
71
  _DEFAULT_APP_DIR = "PLASMA"
72
72
 
73
73
 
74
- def _user_data_dir(app: str = _DEFAULT_APP_DIR) -> str:
75
- """Per-user, per-OS application-data directory (stdlib, no platformdirs)."""
74
+ def user_data_dir(app: str = _DEFAULT_APP_DIR) -> str:
75
+ """Per-user, per-OS application-data directory (stdlib, no platformdirs).
76
+
77
+ Public so other entry points (e.g. ``plasma.desktop_shortcut``) can target
78
+ the same directory a frozen build would resolve to, without themselves
79
+ running frozen.
80
+ """
76
81
  if sys.platform == "win32":
77
82
  base = os.environ.get("LOCALAPPDATA") or os.path.join(
78
83
  os.path.expanduser("~"), "AppData", "Local")
@@ -95,7 +100,7 @@ def _from_env() -> AppContext:
95
100
  if env:
96
101
  return AppContext(home=env)
97
102
  if _is_frozen():
98
- return AppContext(home=_user_data_dir())
103
+ return AppContext(home=user_data_dir())
99
104
  return AppContext(home=os.getcwd())
100
105
 
101
106
 
@@ -14,9 +14,19 @@ Points the shortcut directly at *this* environment's ``plasma`` console-script
14
14
  than re-deriving a `python -m ...` invocation — see the pyshortcuts prototype
15
15
  notes in the packaging discussion for why.
16
16
 
17
+ The shortcut's working directory is pinned to the same per-OS app-data dir a
18
+ frozen PyInstaller build resolves to (``plasma.app_context.user_data_dir``),
19
+ not left to whatever the OS defaults a new shortcut to (``$HOME`` on
20
+ Windows/Linux; an unspecified Terminal default on macOS). Since
21
+ ``plasma.app_context`` falls back to the working directory for a non-frozen
22
+ process, this keeps the Desktop-icon launch and a downloaded prebuilt binary
23
+ reading/writing config from the same place.
24
+
17
25
  plasma-install-shortcut [--name NAME] [--no-terminal] [--no-startmenu]
18
26
 
19
- Requires the ``desktop`` extra (``pip install plasma-app[desktop]``).
27
+ Requires the ``desktop`` extra (``pip install "plasma-app[desktop]"`` —
28
+ quote it, or zsh's glob expansion on ``[...]`` will fail with "no matches
29
+ found").
20
30
  """
21
31
  from __future__ import annotations
22
32
 
@@ -26,6 +36,8 @@ import shutil
26
36
  import sys
27
37
  from pathlib import Path
28
38
 
39
+ from plasma.app_context import user_data_dir
40
+
29
41
  _ICON_DIR = Path(__file__).parent / "resources" / "icons"
30
42
 
31
43
 
@@ -60,6 +72,35 @@ def _plasma_executable() -> str:
60
72
  )
61
73
 
62
74
 
75
+ def _target_home() -> str:
76
+ """Where the shortcut should run PLASMA from — the same per-OS app-data
77
+ dir a frozen build resolves to (``$PLASMA_HOME`` still wins if set,
78
+ matching ``app_context``'s own precedence)."""
79
+ return os.environ.get("PLASMA_HOME") or user_data_dir()
80
+
81
+
82
+ def _launch_script(target_home: str) -> str:
83
+ """Path handed to pyshortcuts as the thing to run.
84
+
85
+ On Windows/Linux, pyshortcuts writes ``target_home`` into the shortcut
86
+ itself (the ``.lnk`` ``WorkingDirectory`` / the ``.desktop`` ``Path=``
87
+ key), so the plain console-script is enough. On macOS, pyshortcuts'
88
+ Terminal/Automator launcher never applies ``working_dir`` at all — so we
89
+ wrap the console-script in a tiny shell script that ``cd``s first.
90
+ """
91
+ exe = _plasma_executable()
92
+ if sys.platform != "darwin":
93
+ return exe
94
+ wrapper = Path(target_home) / "plasma-launch.sh"
95
+ wrapper.write_text(
96
+ "#!/bin/sh\n"
97
+ f'cd "{target_home}" || exit 1\n'
98
+ f'exec "{exe}"\n'
99
+ )
100
+ wrapper.chmod(0o755)
101
+ return str(wrapper)
102
+
103
+
63
104
  def install_shortcut(name="PLASMA", terminal=True, startmenu=True, folder=None):
64
105
  """Create the Desktop (+ Start Menu, if requested) shortcut. Returns the
65
106
  ``pyshortcuts.Shortcut`` namedtuple describing what was written.
@@ -76,19 +117,23 @@ def install_shortcut(name="PLASMA", terminal=True, startmenu=True, folder=None):
76
117
  except ImportError as e:
77
118
  raise ImportError(
78
119
  "pyshortcuts is required to create a desktop shortcut — install "
79
- "it with `pip install plasma-app[desktop]`."
120
+ 'it with `pip install "plasma-app[desktop]"`.'
80
121
  ) from e
81
122
 
123
+ target_home = _target_home()
124
+ os.makedirs(target_home, exist_ok=True)
125
+
82
126
  return make_shortcut(
83
- script=_plasma_executable(),
127
+ script=_launch_script(target_home),
84
128
  name=name,
85
129
  description="PLASMA sensor acquisition dashboard",
86
130
  icon=_icon_path(),
87
131
  terminal=terminal,
88
132
  desktop=True,
89
133
  startmenu=startmenu, # pyshortcuts itself no-ops this on macOS
134
+ working_dir=target_home,
90
135
  folder=folder,
91
- noexe=True, # script is already a standalone console-script binary
136
+ noexe=True, # script is already a standalone console-script/wrapper
92
137
  )
93
138
 
94
139
 
@@ -114,7 +159,13 @@ def main(argv=None):
114
159
  startmenu=not args.no_startmenu,
115
160
  )
116
161
  print(f"Created shortcut: {scut.target}")
117
- print(f" runs: {scut.script}")
162
+ # NOTE: not scut.script/scut.full_script — pyshortcuts splits `script` on
163
+ # its first space to separate a script from trailing arguments, which
164
+ # mangles this when the path itself contains a space (e.g. macOS's
165
+ # "Application Support"). The shortcut it writes is still correct; only
166
+ # the returned namedtuple's script fields are truncated for display.
167
+ print(f" runs: {_plasma_executable()}")
168
+ print(f" in: {scut.working_dir}")
118
169
  print(f" desktop: {scut.desktop_dir}")
119
170
  if scut.startmenu_dir:
120
171
  print(f" menu: {scut.startmenu_dir}")
@@ -39,6 +39,9 @@ def _add_extract_flags(p):
39
39
  p.add_argument('--sniff_threshold', type=float, default=0.90)
40
40
  p.add_argument('--dry_run', action='store_true', default=False)
41
41
  p.add_argument('--strict_ppg', action='store_true', default=False)
42
+ p.add_argument('--include_cdct', action='store_true', default=False,
43
+ help="PPG-device output (ppg/ac legacy/v2) keeps CDCT/init_CDCT/Datetime; "
44
+ "off by default (start time is always in README.txt)")
42
45
  p.add_argument('--note', default="", help="note recorded in the output README")
43
46
 
44
47
 
@@ -68,6 +71,7 @@ def main(argv=None):
68
71
  p_batch.add_argument('--on_format_conflict', choices=CONFLICT_CHOICES, default='warn')
69
72
  p_batch.add_argument('--sniff_threshold', type=float, default=0.90)
70
73
  p_batch.add_argument('--strict_ppg', action='store_true', default=False)
74
+ p_batch.add_argument('--include_cdct', action='store_true', default=False)
71
75
  p_batch.add_argument('--legacy_fs', action='store_true', default=False)
72
76
  p_batch.add_argument('--dry_run', action='store_true', default=False)
73
77
  p_batch.add_argument('--note', default="")
@@ -31,13 +31,13 @@ needed, and where the two disagree the contents win.
31
31
  |---|---|
32
32
  | `auto` | Detect from content, per file. Falls back to the device version if inconclusive. **Default.** |
33
33
  | `version` | Follow `uuid.txt` only (v4.7.0+ → `v2`, otherwise `legacy`). The pre-1.6 behaviour. |
34
- | `legacy` / `v2` / `packed16` / `framed` / `v3` / `block_v2` | Force that layout. |
34
+ | `legacy` / `v2` / `packed16` / `v3` / `block_v2` | Force that layout. |
35
35
 
36
36
  | Sensor | Layouts |
37
37
  |---|---|
38
38
  | PPG | `legacy` 24 B · `v2` 20 B · `packed16` 16 B (no version tie) |
39
- | IMU | `legacy` 30 B record · `v2` 26 B record · `v3` self-describing 4 MiB chunk (no version tie), 2 g / 16384 counts-per-g |
40
- | ECG | `framed` 12 B (pre-v0) · `block_v2` self-describing `ECF2` 4 MiB chunk of 4096 B `ECB2` blocks (no version tie) |
39
+ | IMU | `legacy` 30 B record (wristband) · `v2` 26 B record (wristband) · `v3` self-describing `ACF3` 4 MiB chunk (no version tie; MSense4ECG-Z5G4A chest device) — output columns for `v3` are locked to `SampleSequence`/`RtcTickEstBlock`/`RtcTickEst`/`AccX`/`AccY`/`AccZ` |
40
+ | ECG | `block_v2` self-describing `ECF2` 4 MiB chunk of 4096 B `ECB2` blocks (no version tie) — the only ECG layout; output columns are locked to `SampleIndex`/`RtcTick`/`ECG`/`ETAG`/`PTAG` |
41
41
 
42
42
  **Cross-check against uuid.txt** reports when detection and the version file
43
43
  disagree. It is off by default and does not override detection: use
@@ -46,6 +46,14 @@ disagree. It is off by default and does not override detection: use
46
46
  **Strict record validation**: raise on a record that fails its integrity check
47
47
  (packed16 reserved bits, ECG CRC, a partial trailing record) instead of dropping
48
48
  it and reporting a count.
49
+
50
+ **Include CDCT/Datetime** (off by default): PPG/IMU `legacy`/`v2`/`packed16`
51
+ output no longer carries `CDCT`/`init_CDCT`/`Datetime` unless this is ticked —
52
+ `ecg:block_v2` and IMU `v3` never had these columns to begin with (see their
53
+ locked schemas above). Every PPG-device file's start time (UTC, from its
54
+ filename) is always recorded in `README.txt` regardless of this setting, so
55
+ turn it on only if you need the per-row columns themselves (e.g. to feed
56
+ `clocksync.py`, which is CSV/`Counter`+`CDCT`-based).
49
57
  """
50
58
 
51
59
 
@@ -71,6 +79,12 @@ class ExtractionOptions:
71
79
  on_format_conflict: str = "warn"
72
80
  strict_ppg: bool = False
73
81
  force_new_format: bool = False
82
+ # Off by default: PPG-device output (ppg:*, ac:legacy/v2) no longer carries
83
+ # CDCT/init_CDCT/Datetime. ecg:block_v2 and ac:v3 never had these columns
84
+ # regardless of this setting (see formats.py's locked schemas). Each
85
+ # PPG-device file's start time is always written to README.txt instead —
86
+ # see DataExtractor.write_provenance.
87
+ include_cdct: bool = False
74
88
  sniff_threshold: float = field(default=0.90, metadata={"panel": False})
75
89
  dry_run: bool = field(default=False, metadata={"panel": False})
76
90
 
@@ -96,6 +110,7 @@ class ExtractionOptions:
96
110
  on_format_conflict=args.on_format_conflict,
97
111
  strict_ppg=args.strict_ppg,
98
112
  force_new_format=args.force_new_format,
113
+ include_cdct=args.include_cdct,
99
114
  sniff_threshold=args.sniff_threshold,
100
115
  dry_run=args.dry_run,
101
116
  )
@@ -177,6 +177,8 @@ class DataExtractor():
177
177
  self.malformed = 0
178
178
  self.dropped = 0
179
179
  self.out_paths = []
180
+ self.anchor_block_summaries = [] # (basename, first, last, n_ok, n_bad) — README-only
181
+ self.ppg_file_start_times = [] # (basename, UTC datetime str) — README-only, PPG-device files only
180
182
 
181
183
  self.encoding_alias = self.get_encoding_alias() if self.df is not None else {}
182
184
 
@@ -197,6 +199,7 @@ class DataExtractor():
197
199
  file.write(f"Cross-check against uuid.txt = {options.validate_with_uuid}"
198
200
  f" (on conflict: {options.on_format_conflict})\n")
199
201
  file.write(f"Strict record validation = {options.strict_ppg}\n")
202
+ file.write(f"Include CDCT/init_CDCT/Datetime (PPG-device output) = {options.include_cdct}\n")
200
203
  file.write(f"Detection threshold = {options.sniff_threshold}\n")
201
204
  file.write("I m-sense with YAMS at https://github.com/SenSE-Lab-OSU/YAMS\n")
202
205
  uuid_path = os.path.join(self.in_dir, "uuid.txt")
@@ -259,6 +262,22 @@ class DataExtractor():
259
262
  if conflicts:
260
263
  file.write(f"uuid.txt conflicts = {len(conflicts)} "
261
264
  f"(content used unless on_format_conflict=trust_uuid)\n")
265
+ if self.anchor_block_summaries:
266
+ file.write("\n--- AC/ECG block metadata (per ACB1/ECB2 block header, "
267
+ 'BLOCK_META_FMT = "<4sIII": magic, AnchorCounter '
268
+ "(reserved_timer_output / first_rtc_tick), "
269
+ "OrigCounter base (first_sample_sequence / first_sample_index), "
270
+ "block_crc32) ---\n")
271
+ for basename, first, last, n_ok, n_bad in self.anchor_block_summaries:
272
+ file.write(f"{basename}: AnchorCounter {first}..{last} "
273
+ f"({n_ok} block(s) ok, {n_bad} bad)\n")
274
+ if self.ppg_file_start_times:
275
+ # Recorded regardless of `include_cdct`: once that option is off
276
+ # (the default), this is the only place a PPG-device file's
277
+ # start time (its filename-derived t0) still shows up.
278
+ file.write("\n--- PPG-device file start times (UTC) ---\n")
279
+ for basename, dt in sorted(self.ppg_file_start_times):
280
+ file.write(f"{basename}: {dt}\n")
262
281
 
263
282
  def resolve(self, full_path, sensor):
264
283
  res = detect.resolve(
@@ -273,9 +292,23 @@ class DataExtractor():
273
292
 
274
293
  def read_file(self, full_path, sensor):
275
294
  res = self.resolve(full_path, sensor)
276
- df, dt = formats.read_bin(full_path, res.spec, strict=self.strict)
295
+ df, dt = formats.read_bin(full_path, res.spec, strict=self.strict,
296
+ include_cdct=self.options.include_cdct)
277
297
  self.malformed += df.attrs.get('malformed_records', 0)
278
298
  self.dropped += df.attrs.get('dropped_samples', 0)
299
+ if res.spec.key in ("ac:v3", "ecg:block_v2"):
300
+ self.anchor_block_summaries.append((
301
+ os.path.basename(full_path),
302
+ df.attrs.get("anchor_counter_first"),
303
+ df.attrs.get("anchor_counter_last"),
304
+ df.attrs.get("n_blocks_ok", 0),
305
+ df.attrs.get("n_blocks_bad", 0),
306
+ ))
307
+ else:
308
+ # A flat PPG-device format (ppg:*, ac:legacy/v2) — the two
309
+ # container formats (ac:v3, ecg:block_v2) belong to the
310
+ # MSense4ECG chest device and are excluded here.
311
+ self.ppg_file_start_times.append((os.path.basename(full_path), dt))
279
312
  return df, res
280
313
 
281
314
  def extract_csv(self, search_prefix, file_name, id=-1):
@@ -326,7 +359,8 @@ class DataExtractor():
326
359
  # several distinct recordings, so a legitimate gap *between*
327
360
  # recordings is never mistaken for a dropped-sample run within one.
328
361
  counter_validity_check(chunk, spec)
329
- chunk = _finish_chunk(chunk, is_ac=is_ac, spec=spec)
362
+ chunk = _finish_chunk(chunk, is_ac=is_ac, spec=spec,
363
+ include_cdct=self.options.include_cdct)
330
364
  if whole_frame:
331
365
  pieces.append(chunk)
332
366
  else:
@@ -353,16 +387,21 @@ class DataExtractor():
353
387
  one-chunk "session") and, for any session spanning more than one chunk,
354
388
  restitch CDCT as one continuous clock anchored at that session's t0.
355
389
 
356
- Container formats need no CDCT recompute:
357
- - `ac:v3` (ACF3) anchors every sample to a session-continuous RTC tick
358
- per block internally. Chunks are re-ordered by `chunk_index` and a
359
- `first_sample_sequence` discontinuity at a chunk boundary (firmware
360
- drop, or a missing chunk) is reported.
361
- - `ecg:block_v2` (ECF2) makes `CDCT = t0 + Counter/512` where `Counter`
362
- is the recording-local sample ordinal that already advances across
363
- chunks — so plain concatenation is continuous. ECF2 chunks are
364
- re-ordered by `chunk_index` and split by `recording_id`, and a gap
365
- in `chunk_index` or `Counter` is reported (not silently joined).
390
+ Container formats need no CDCT recompute — both are locked to
391
+ ECG_EXTRACTION_HANDOFF.md's schemas and have no CDCT column at all:
392
+ - `ac:v3` (ACF3) columns are `SampleSequence`, `RtcTickEstBlock`,
393
+ `RtcTickEst`, `AccX`, `AccY`, `AccZ`. Chunks are re-ordered by
394
+ `chunk_index`; a `first_sample_sequence` discontinuity at a chunk
395
+ boundary (firmware drop, or a missing chunk) is reported, and
396
+ `RtcTickEst` (the piecewise-linear per-sample tick ramp) is built
397
+ across the whole session's blocks — see `_stitch_ac_v3_chunks`.
398
+ - `ecg:block_v2` (ECF2) has no CDCT at all: its output is locked to
399
+ ECG_EXTRACTION_HANDOFF.md's schema (`SampleIndex`, `RtcTick`, `ECG`,
400
+ `ETAG`, `PTAG`). `SampleIndex` is the recording-local sample ordinal
401
+ that already advances across chunks, so plain concatenation is
402
+ continuous. ECF2 chunks are re-ordered by `chunk_index` and split by
403
+ `recording_id`, and a gap in `chunk_index` or `SampleIndex` is
404
+ reported (not silently joined).
366
405
  """
367
406
  session_dfs, spec = self._collect_session_frames(path, prefix)
368
407
  if not session_dfs:
@@ -405,7 +444,10 @@ class DataExtractor():
405
444
  session_dfs.append(dfs[0])
406
445
  else:
407
446
  combined = pd.concat(dfs, ignore_index=True)
408
- if spec.read_file is None: # flat per-record formats only
447
+ # flat per-record formats only, and only when CDCT is wanted —
448
+ # `read_bin` already omitted it per-chunk when include_cdct is
449
+ # off (the default), so re-stitching it here would put it back.
450
+ if spec.read_file is None and self.options.include_cdct:
409
451
  combined = formats.recompute_cdct(combined, spec, t0)
410
452
  session_dfs.append(combined)
411
453
 
@@ -442,11 +484,44 @@ def gather_files_by_prefix(prefix: str, path):
442
484
  return all_files
443
485
 
444
486
 
487
+ def _piecewise_linear_ac_tick(block_ticks, block_n_samples):
488
+ """Per-sample `RtcTickEst`: within each block, ramp linearly from that
489
+ block's own `RtcTickEstBlock` anchor toward the *next* block's anchor,
490
+ dividing the actual elapsed ticks by that block's actual sample count (not
491
+ the nominal 680) — so real per-block rate variation is preserved, and the
492
+ ramp is exactly continuous at block boundaries (block i+1 starts exactly
493
+ at its own anchor). `block_ticks`/`block_n_samples` must already span the
494
+ whole session (every chunk's blocks, in `chunk_index` order — see
495
+ `_stitch_ac_v3_chunks`), since the last block of one chunk ramps toward
496
+ the first block of the next. The session's last block has no next anchor
497
+ to ramp toward; it extrapolates using the previous block's local rate.
498
+ Ported from ECG_EXTRACTION_HANDOFF.md's `extract_ecg_ac_v3.py`.
499
+ """
500
+ n_blocks = len(block_ticks)
501
+ segments = []
502
+ prev_rate = 0.0
503
+ for i in range(n_blocks):
504
+ n = block_n_samples[i]
505
+ if i < n_blocks - 1:
506
+ rate = (block_ticks[i + 1] - block_ticks[i]) / n
507
+ else:
508
+ rate = prev_rate
509
+ segments.append(block_ticks[i] + rate * np.arange(n, dtype=np.float64))
510
+ prev_rate = rate
511
+ return np.concatenate(segments) if segments else np.array([], dtype=np.float64)
512
+
513
+
445
514
  def _stitch_ecb2_chunks(dfs):
446
515
  """Order ECF2 chunk DataFrames by `recording_id` then `chunk_index`,
447
- report any `chunk_index` or `Counter` discontinuity, and return one
448
- concatenated DataFrame per recording (CDCT is already continuous — see
449
- `formats._read_ecf2`)."""
516
+ report any `chunk_index` or `SampleIndex` discontinuity, and return one
517
+ concatenated DataFrame per recording.
518
+
519
+ Unlike `_stitch_ac_v3_chunks`, no global `Counter` offset is added:
520
+ `SampleIndex` is already the recording-local sample ordinal, continuous by
521
+ construction across every chunk of one `recording_id` (see
522
+ `formats._read_ecf2`), so plain concatenation is the whole job — the
523
+ output column set stays locked to ECG_EXTRACTION_HANDOFF.md's schema.
524
+ """
450
525
  by_recording = {}
451
526
  for df in dfs:
452
527
  by_recording.setdefault(df.attrs.get("recording_id"), []).append(df)
@@ -471,13 +546,23 @@ def _stitch_ecb2_chunks(dfs):
471
546
 
472
547
 
473
548
  def _stitch_ac_v3_chunks(dfs):
474
- """Order ACF3 chunk DataFrames by `chunk_index` and report a
549
+ """Order ACF3 chunk DataFrames by `chunk_index`, report a
475
550
  `first_sample_sequence` discontinuity at a chunk boundary — a firmware drop
476
551
  ("a larger difference records the number of missing samples", per the
477
- format doc) or a missing chunk. CDCT is already absolute per chunk (each
478
- ACB1 block carries its own RTC anchor), so this only concatenates."""
552
+ format doc) or a missing chunk — and return one concatenated DataFrame for
553
+ the session with `RtcTickEst` added.
554
+
555
+ `RtcTickEst` (the piecewise-linear per-sample tick ramp) is computed here,
556
+ not per-chunk in `formats._read_ac_v3`, because it can span a chunk
557
+ boundary: the last block of one chunk ramps toward the first block of the
558
+ next, so the full session's `(block_tick, n_samples)` list — gathered from
559
+ every chunk's `block_ticks`/`block_n_samples` attrs, in `chunk_index`
560
+ order — must be known first. Output columns otherwise stay locked to
561
+ ECG_EXTRACTION_HANDOFF.md's schema: no global `Counter` offset is added.
562
+ """
479
563
  group = sorted(dfs, key=lambda d: d.attrs.get("chunk_index", 0))
480
564
  prev = None
565
+ session_block_ticks, session_block_n = [], []
481
566
  for d in group:
482
567
  ci = d.attrs.get("chunk_index", 0)
483
568
  if ci == 0 and d.attrs.get("first_seq") not in (None, 0):
@@ -494,7 +579,11 @@ def _stitch_ac_v3_chunks(dfs):
494
579
  print(f"AC session: {gap} sample(s) dropped at the chunk "
495
580
  f"{ci} boundary")
496
581
  prev = d
497
- return pd.concat(group, ignore_index=True)
582
+ session_block_ticks.extend(d.attrs.get("block_ticks", []))
583
+ session_block_n.extend(d.attrs.get("block_n_samples", []))
584
+ combined = pd.concat(group, ignore_index=True)
585
+ combined["RtcTickEst"] = _piecewise_linear_ac_tick(session_block_ticks, session_block_n)
586
+ return combined
498
587
 
499
588
 
500
589
  def counter_validity_check(df: pd.DataFrame, spec=None):
@@ -506,8 +595,19 @@ def counter_validity_check(df: pd.DataFrame, spec=None):
506
595
  if spec is None:
507
596
  print("pass counter check: N/A (no format resolved)")
508
597
  return
509
- # The readers append CDCT/init_CDCT, so the last column is not the counter.
510
- counter_columns = df[['Counter']] if 'Counter' in df.columns else df.iloc[:, -1:]
598
+ # Flat ppg/ac layouts (legacy/v2/packed16) use 'Counter'. The two
599
+ # container formats are locked to ECG_EXTRACTION_HANDOFF.md's schema and
600
+ # have their own analogous column instead: ecg:block_v2's 'SampleIndex'
601
+ # and ac:v3's 'SampleSequence'. Absent all three the last column is not
602
+ # the counter either.
603
+ if 'Counter' in df.columns:
604
+ counter_columns = df[['Counter']]
605
+ elif 'SampleIndex' in df.columns:
606
+ counter_columns = df[['SampleIndex']]
607
+ elif 'SampleSequence' in df.columns:
608
+ counter_columns = df[['SampleSequence']]
609
+ else:
610
+ counter_columns = df.iloc[:, -1:]
511
611
  counter_arr = numpy.array(counter_columns).flatten()
512
612
  diff_arr = numpy.diff(counter_arr)
513
613
  step = spec.tick_step
@@ -520,12 +620,12 @@ def counter_validity_check(df: pd.DataFrame, spec=None):
520
620
  print("and number of non matching samples: " + str(numpy.count_nonzero(check_array == 0)))
521
621
 
522
622
 
523
- def _finish_chunk(data_set, *, is_ac, spec):
524
- """Add the human-readable Datetime column and (for AC) convert counts to
525
- g — replacing a Python `datetime.fromtimestamp()` + `.strftime()` call per
526
- row, the dominant cost of the old single-shot extraction (millions of
527
- interpreted calls for a long high-rate recording), not the CSV write it
528
- was blamed for.
623
+ def _finish_chunk(data_set, *, is_ac, spec, include_cdct):
624
+ """Add the human-readable Datetime column (when `include_cdct`) and (for
625
+ AC) convert counts to g — replacing a Python `datetime.fromtimestamp()` +
626
+ `.strftime()` call per row, the dominant cost of the old single-shot
627
+ extraction (millions of interpreted calls for a long high-rate
628
+ recording), not the CSV write it was blamed for.
529
629
 
530
630
  Deliberately NOT `pd.to_datetime(...).dt.strftime(...)`: benchmarked at
531
631
  ~2.9s for 2M rows vs. ~2.9s for the original per-row loop — pandas'
@@ -536,16 +636,31 @@ def _finish_chunk(data_set, *, is_ac, spec):
536
636
  (`numpy.datetime_as_string`), then two vectorized character replaces to
537
637
  turn ISO-8601 into this project's on-disk format — ~0.6s for the same 2M
538
638
  rows, ~5x the original loop, verified to produce byte-identical strings.
639
+
640
+ ecg:block_v2 and ac:v3 output have no `CDCT` column at all — both are
641
+ locked to ECG_EXTRACTION_HANDOFF.md's schemas (`SampleIndex`/`RtcTick`/
642
+ `ECG`/`ETAG`/`PTAG` and `SampleSequence`/`RtcTickEstBlock`/`RtcTickEst`/
643
+ `AccX`/`AccY`/`AccZ` respectively — `AccX/Y/Z` are already in g, converted
644
+ in `formats._read_ac_v3`), so this is a no-op for both regardless of
645
+ `include_cdct` rather than fabricating a Datetime column or re-converting
646
+ units. For the flat ppg/ac formats, `include_cdct=False` (the default)
647
+ means `read_bin` already omitted `CDCT`/`init_CDCT` (see
648
+ `ExtractionOptions.include_cdct`), so this skips Datetime too — unit
649
+ conversion for AC still runs either way, since it has nothing to do with
650
+ CDCT.
539
651
  """
540
- try:
541
- secs = data_set['CDCT'].to_numpy()
542
- if not np.isfinite(secs).all():
543
- raise ValueError("non-finite CDCT value(s) — can't convert to a timestamp")
544
- iso = np.datetime_as_string(secs.astype(np.int64).astype('datetime64[s]'), unit='s')
545
- data_set['Datetime'] = np.char.replace(np.char.replace(iso, '-', '/'), 'T', ' ')
546
- except Exception as e:
547
- print(str(e))
548
- data_set['Datetime'] = -1
652
+ if spec is not None and spec.key in ("ecg:block_v2", "ac:v3"):
653
+ return data_set
654
+ if include_cdct:
655
+ try:
656
+ secs = data_set['CDCT'].to_numpy()
657
+ if not np.isfinite(secs).all():
658
+ raise ValueError("non-finite CDCT value(s) — can't convert to a timestamp")
659
+ iso = np.datetime_as_string(secs.astype(np.int64).astype('datetime64[s]'), unit='s')
660
+ data_set['Datetime'] = np.char.replace(np.char.replace(iso, '-', '/'), 'T', ' ')
661
+ except Exception as e:
662
+ print(str(e))
663
+ data_set['Datetime'] = -1
549
664
  if is_ac:
550
665
  print("perform unit conversion for IMU")
551
666
  data_set = unit_conversion_ac(data_set, spec)
@@ -553,15 +668,12 @@ def _finish_chunk(data_set, *, is_ac, spec):
553
668
 
554
669
 
555
670
  def unit_conversion_ac(data_set, spec=None):
556
- """Raw counts -> g. The v3 (ACF3) layout documents its own scale; legacy/v2
557
- (the wristband) keep the original conversion so their output is unchanged.
671
+ """Raw counts -> g, for the legacy/v2 (wristband) IMU layouts — kept
672
+ unchanged. ac:v3's own scale is applied in `formats._read_ac_v3` instead
673
+ (its output is already in g), since `_finish_chunk` skips this call for it.
558
674
  """
559
- if spec is not None and spec.key == "ac:v3":
560
- for c in ['AccX', 'AccY', 'AccZ']:
561
- data_set[c] = data_set[c] / formats.AC_V3_COUNTS_PER_G
562
- else:
563
- for c in ['AccX', 'AccY', 'AccZ']:
564
- data_set[c] = data_set[c] / (2**16 - 1) * 8
675
+ for c in ['AccX', 'AccY', 'AccZ']:
676
+ data_set[c] = data_set[c] / (2**16 - 1) * 8
565
677
  return data_set
566
678
 
567
679