framelock 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. framelock-0.3.0/.github/workflows/ci.yml +21 -0
  2. framelock-0.3.0/.github/workflows/release.yml +22 -0
  3. framelock-0.3.0/.gitignore +17 -0
  4. framelock-0.3.0/.pre-commit-config.yaml +13 -0
  5. framelock-0.3.0/.python-version +1 -0
  6. framelock-0.3.0/AGENTS.md +121 -0
  7. framelock-0.3.0/CLAUDE.md +1 -0
  8. framelock-0.3.0/LICENSE +21 -0
  9. framelock-0.3.0/PKG-INFO +283 -0
  10. framelock-0.3.0/README.md +251 -0
  11. framelock-0.3.0/docs/CAMERA_METADATA.md +215 -0
  12. framelock-0.3.0/docs/CHIRP_BENCHMARK.md +34 -0
  13. framelock-0.3.0/docs/MEASUREMENTS.md +77 -0
  14. framelock-0.3.0/docs/OUTPUTS.md +337 -0
  15. framelock-0.3.0/docs/SYNCHRONIZATION.md +220 -0
  16. framelock-0.3.0/docs/TAGS.md +129 -0
  17. framelock-0.3.0/docs/chirps/README.md +34 -0
  18. framelock-0.3.0/docs/chirps/bubble_code.wav +0 -0
  19. framelock-0.3.0/docs/chirps/glass_plink.wav +0 -0
  20. framelock-0.3.0/docs/chirps/hidden_water.wav +0 -0
  21. framelock-0.3.0/docs/chirps/mls.wav +0 -0
  22. framelock-0.3.0/docs/chirps/natural_water.wav +0 -0
  23. framelock-0.3.0/docs/chirps/reference.wav +0 -0
  24. framelock-0.3.0/docs/chirps/sparkle.wav +0 -0
  25. framelock-0.3.0/docs/chirps/sweep.wav +0 -0
  26. framelock-0.3.0/docs/chirps/velvet.wav +0 -0
  27. framelock-0.3.0/examples/01_single_oak.py +22 -0
  28. framelock-0.3.0/examples/02_oak_plus_iphone.py +69 -0
  29. framelock-0.3.0/examples/03_zoom_automation.py +25 -0
  30. framelock-0.3.0/examples/04_live_control.py +42 -0
  31. framelock-0.3.0/examples/05_custom_chirp.py +38 -0
  32. framelock-0.3.0/examples/06_offline_analysis.py +51 -0
  33. framelock-0.3.0/examples/07_mobile_rig.py +88 -0
  34. framelock-0.3.0/examples/08_air_conditions.py +65 -0
  35. framelock-0.3.0/examples/09_multi_iphone.py +78 -0
  36. framelock-0.3.0/examples/10_odometry.py +114 -0
  37. framelock-0.3.0/examples/11_wall_tags.py +91 -0
  38. framelock-0.3.0/framelock/__init__.py +115 -0
  39. framelock-0.3.0/framelock/__main__.py +6 -0
  40. framelock-0.3.0/framelock/_compat.py +49 -0
  41. framelock-0.3.0/framelock/assets/chirps/00_reference_linear_ramp_chirp.wav +0 -0
  42. framelock-0.3.0/framelock/assets/chirps/01_hidden_chirp_water_ringtone.wav +0 -0
  43. framelock-0.3.0/framelock/assets/chirps/02_glass_plink_hidden_probe.wav +0 -0
  44. framelock-0.3.0/framelock/assets/chirps/03_bubble_code_microchirps.wav +0 -0
  45. framelock-0.3.0/framelock/assets/chirps/04_tiny_sparkle_notification_good_sync.wav +0 -0
  46. framelock-0.3.0/framelock/assets/chirps/05_natural_water_wood_phrase_no_obvious_ramp.wav +0 -0
  47. framelock-0.3.0/framelock/audioalign.py +1068 -0
  48. framelock-0.3.0/framelock/chirps.py +679 -0
  49. framelock-0.3.0/framelock/cli.py +349 -0
  50. framelock-0.3.0/framelock/clock.py +53 -0
  51. framelock-0.3.0/framelock/controls.py +247 -0
  52. framelock-0.3.0/framelock/correspondence.py +765 -0
  53. framelock-0.3.0/framelock/diagnostics.py +643 -0
  54. framelock-0.3.0/framelock/odometry.py +374 -0
  55. framelock-0.3.0/framelock/preview.py +591 -0
  56. framelock-0.3.0/framelock/registration/__init__.py +183 -0
  57. framelock-0.3.0/framelock/registration/_se3.py +81 -0
  58. framelock-0.3.0/framelock/registration/acoustic.py +517 -0
  59. framelock-0.3.0/framelock/registration/board.py +571 -0
  60. framelock-0.3.0/framelock/registration/cli.py +268 -0
  61. framelock-0.3.0/framelock/registration/store.py +252 -0
  62. framelock-0.3.0/framelock/registration/tagcli.py +127 -0
  63. framelock-0.3.0/framelock/registration/tags.py +793 -0
  64. framelock-0.3.0/framelock/registration/visual.py +171 -0
  65. framelock-0.3.0/framelock/registration/wave.py +1242 -0
  66. framelock-0.3.0/framelock/session.py +1781 -0
  67. framelock-0.3.0/framelock/soundspeed.py +370 -0
  68. framelock-0.3.0/framelock/sources/__init__.py +6 -0
  69. framelock-0.3.0/framelock/sources/base.py +177 -0
  70. framelock-0.3.0/framelock/sources/mic.py +356 -0
  71. framelock-0.3.0/framelock/sources/oak.py +1520 -0
  72. framelock-0.3.0/framelock/sources/phonedata.py +559 -0
  73. framelock-0.3.0/framelock/sources/rtsp.py +778 -0
  74. framelock-0.3.0/framelock/sync.py +242 -0
  75. framelock-0.3.0/framelock/viz.py +931 -0
  76. framelock-0.3.0/pyproject.toml +92 -0
  77. framelock-0.3.0/tests/test_audioalign.py +580 -0
  78. framelock-0.3.0/tests/test_chirps.py +106 -0
  79. framelock-0.3.0/tests/test_controls.py +55 -0
  80. framelock-0.3.0/tests/test_correspondence.py +565 -0
  81. framelock-0.3.0/tests/test_oakdata.py +183 -0
  82. framelock-0.3.0/tests/test_odometry.py +286 -0
  83. framelock-0.3.0/tests/test_phonedata.py +360 -0
  84. framelock-0.3.0/tests/test_preview.py +247 -0
  85. framelock-0.3.0/tests/test_registration.py +1392 -0
  86. framelock-0.3.0/tests/test_session.py +1025 -0
  87. framelock-0.3.0/tests/test_soundspeed.py +112 -0
  88. framelock-0.3.0/tests/test_sync.py +217 -0
  89. framelock-0.3.0/tests/test_viz.py +205 -0
  90. framelock-0.3.0/uv.lock +982 -0
@@ -0,0 +1,21 @@
1
+ name: ci
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ test:
10
+ runs-on: ubuntu-latest
11
+ steps:
12
+ - uses: actions/checkout@v4
13
+ - uses: astral-sh/setup-uv@v5
14
+ with:
15
+ python-version: "3.12"
16
+ - name: System libraries (PortAudio for sounddevice, GL for OpenCV)
17
+ run: sudo apt-get update && sudo apt-get install -y libportaudio2 libgl1
18
+ - name: Lint
19
+ run: uv run ruff check .
20
+ - name: Tests
21
+ run: uv run pytest -q
@@ -0,0 +1,22 @@
1
+ name: release
2
+
3
+ on:
4
+ push:
5
+ tags: ["v*"]
6
+
7
+ jobs:
8
+ pypi:
9
+ runs-on: ubuntu-latest
10
+ environment: pypi
11
+ permissions:
12
+ contents: read # an explicit permissions block resets the rest
13
+ id-token: write # PyPI Trusted Publishing (OIDC) — no token stored
14
+ steps:
15
+ - uses: actions/checkout@v4
16
+ - uses: astral-sh/setup-uv@v5
17
+ with:
18
+ python-version: "3.12"
19
+ - name: Build
20
+ run: uv build
21
+ - name: Publish to PyPI
22
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,17 @@
1
+ # Python-generated files
2
+ __pycache__/
3
+ *.py[oc]
4
+ build/
5
+ dist/
6
+ wheels/
7
+ *.egg-info
8
+
9
+ # Virtual environments
10
+ .venv
11
+
12
+ # Recorded sessions (data, not code)
13
+ recordings/
14
+
15
+ # Tool caches / OS noise
16
+ .cache/
17
+ .DS_Store
@@ -0,0 +1,13 @@
1
+ # uv run pre-commit install (once per clone)
2
+ repos:
3
+ - repo: https://github.com/astral-sh/ruff-pre-commit
4
+ rev: v0.15.14
5
+ hooks:
6
+ - id: ruff-check
7
+ - repo: https://github.com/pre-commit/pre-commit-hooks
8
+ rev: v5.0.0
9
+ hooks:
10
+ - id: check-merge-conflict
11
+ - id: end-of-file-fixer
12
+ - id: trailing-whitespace
13
+ args: [--markdown-linebreak-ext=md]
@@ -0,0 +1 @@
1
+ 3.12
@@ -0,0 +1,121 @@
1
+ # framelock (GitHub: ryanrudes/framelock; local folder: oakd_record)
2
+
3
+ Synchronized multi-camera recording for multi-view pose estimation
4
+ (GVHMR): OAK-D (DepthAI v3) + iPhones running the user's own **iRTSP**
5
+ app + host mic — one session folder, sub-millisecond shared clock,
6
+ odometry, rig registration, corresponded exports, 3D playback.
7
+
8
+ Python 3.12, `uv`. Run everything through `uv run` (pytest, the
9
+ `framelock <subcommand>` CLI — record/align/correspond/preview/
10
+ register/tags/report/viz; `framelock-<sub>` aliases remain —
11
+ and `examples/*.py`). Full test suite: `uv run pytest`
12
+ (~70 s, must stay green). File/column reference: `docs/OUTPUTS.md`.
13
+
14
+ ## The one idea everything hangs on
15
+
16
+ Every sample from every device lands on ONE host-monotonic timeline,
17
+ and every claim gets **verified by an independent estimator** before
18
+ it's trusted. When adding anything, ask: what clock is this timestamp
19
+ on, and what cross-checks it? Never trust a single estimator when two
20
+ can vote (screen-size vs ARKit vs stereo; acoustic bias vs PRFT vs
21
+ motion-envelope; phone stillness testimony vs static-camera testimony).
22
+
23
+ ## Clock model (do not break)
24
+
25
+ - Package sync domain: Python `time.monotonic()` seconds. Epoch via a
26
+ per-session frozen `ClockMap`.
27
+ - OAK: `getTimestamp() − dai_offset`; dai uses C++ steady_clock which
28
+ DIVERGES from time.monotonic() on macOS by accumulated sleep (days!)
29
+ — the bracketed-median offset in `oak.py` is mandatory.
30
+ - Phone video: `ts = pts + min(arrival − pts) + offset`; acoustic label
31
+ bias measured per take (`audio_alignment.json`), applied at export.
32
+ - Phone odometry: the iRTSP app stamps everything from ONE anchor per
33
+ session; `unix_ts` == the video's RTCP-SR (PRFT) axis bit-for-bit, so
34
+ odometry → pts axis is a measured constant (`wall_to_pts`), never an
35
+ estimate. Contract: `irtsp-support/INTEGRATION.md`;
36
+ client: `ryanrudes/irtsp-python` (>=0.2.0, `Pose.discontinuity`).
37
+
38
+ ## Layout
39
+
40
+ - `framelock/session.py` — record loop, keys (r/q/c/b/g), auto
41
+ align/register/export at session end.
42
+ - `framelock/sources/` — `base.py` (Source protocol), `oak.py`
43
+ (+`data=("imu","depth","features","system","vio")`), `rtsp.py` +
44
+ `phonedata.py` (iRTSP odometry/depth recorder), `mic.py` (chirps,
45
+ `chirps.csv`, `chirp_template.npz`).
46
+ - `framelock/registration/` — a package, one module per concern:
47
+ `_se3.py` (rotation helpers + frame constants), `board.py` (specs,
48
+ rendering, detection, the live 'b' display, `board_display.csv`),
49
+ `wave.py` (the whole-rig wave solver: continuous-time co-sightings OR
50
+ verified-static board, timed anchors, IMU stillness witness, gravity
51
+ leveling, size votes, world-origin rebasing), `tags.py` (printed wall
52
+ tags: surveyed by localized devices, then phone drift anchors +
53
+ camera rescue), `acoustic.py`
54
+ (stop-and-chirp solver + ritual), `visual.py` (offline screen-board),
55
+ `store.py` (`registration.json`, `transform_at`, coverage), `cli.py`.
56
+ The public surface is re-exported from `framelock.registration`.
57
+ - `framelock/audioalign.py` (GCC-PHAT bias/drift), `correspondence.py`
58
+ (timelines, corresponded export, localization-gated trim),
59
+ `odometry.py` (pose resample, gyro-vs-flow verifier), `viz.py`
60
+ (viser), `chirps.py`, `_compat.py` (macOS objc-warning muffler).
61
+
62
+ ## Hard-won pitfalls (each cost a live take or a review to find)
63
+
64
+ - rich auto-highlight injects version-dependent ANSI codes → every
65
+ Console uses `highlight=False`. viser pins `rich<15`.
66
+ - BNO086/RVC2 + depthai 3.7.1: `*_CALIBRATED` IMU variants emit ZERO
67
+ reports — use `*_RAW` (Luxonis' own examples do). Gyro ≤400 Hz. And a
68
+ second `enableIMUSensor` call CLOBBERS the first (measured twice) —
69
+ configure ALL report types in ONE call. RAW reports live in the IMU
70
+ CHIP frame: on the OAK-D the BNO086 sits rotated 180° about camera z
71
+ (measured via solved-pose × parked-accel; assuming identity leveled
72
+ the world 171° upside down). `_R_CAM_FROM_IMU` in
73
+ `registration/wave.py`; a device-recorded `IMU_to_CAM_A` in
74
+ calibration.json overrides it.
75
+ - OAK monos get BLINDED by a bright screen at locked exposure (34%
76
+ saturation measured) — the board console shows it live; dim screen.
77
+ - Frame timestamps are END-of-exposure (`exposure_us` column recorded
78
+ for mid-exposure correction). RVC2 encoder ceiling ~60 fps total, but
79
+ the ISP-pass budget bites FIRST: every extra requestOutput shaves fps
80
+ off every stream at once (previews cost 3.3, depth 1.5 — measured;
81
+ see docs/SYNCHRONIZATION.md). Previews now decode host-side from the
82
+ recording taps (free); features ride the stereo-left stream; request
83
+ the warning's prescribed sustainable fps for GAPLESS capture. Mixed
84
+ per-camera output rates stall the camera. The exporter pairs
85
+ multi-stream sources to joint instants so drops never tear a device.
86
+ - Planar PnP: use IPPE + ambiguity rejection; near-frontal + far views
87
+ of a small board are degenerate (weights + 80 px gate handle it).
88
+ - ARKit odom DRIFTS silently when feature-starved (bright screen in
89
+ view; 30% LIMITED tracking measured) — no discontinuity event fires
90
+ for drift. Static-camera testimony outranks phone testimony; per-spot
91
+ anchors absorb wander. Cameras without odometry must sight the board
92
+ from their FINAL pose (their IMU gates this when present).
93
+ - Screen-derived board size lies ~15% on notched/scaled Macs — seed
94
+ only; authority order: ruler > stereo/ARKit votes > screen.
95
+ - `generateImage` carves margins from the requested size (use
96
+ `board_image()`); ChArUco `square_mm` = chessboard square edge,
97
+ aruco_grid `square_mm` = marker edge.
98
+ - cv2's key queue is process-global; keys are routed from all windows.
99
+ - PyAV RTSP: never use its `timeout=` interrupt callback (GIL
100
+ starvation drops the stream); libav `timeout` option only.
101
+ - Same-template chirp search windows must stay disjoint
102
+ (`RegistrationRitual.GAP_S` ↔ `_TOA_SEARCH_*`).
103
+
104
+ ## Conventions
105
+
106
+ - ARKit: camera = `ARCamera.transform`, gravity world +Y up;
107
+ `R_cv = R_arkit @ diag(1,−1,−1)`; registration frames z-up
108
+ (`R_WORLD_FROM_ODOM` in `registration.py` is the single source).
109
+ - dai extrinsics are centimeters — `calibration.json` stores meters.
110
+ - Tests are synthetic-physics: build takes from ground truth, assert
111
+ recovery (see `tests/test_registration.py` `_wave_rig`,
112
+ `_write_synthetic_take`). New solvers need a GT-recovery test.
113
+ - Live testing: user prefers `ChirpKind.BUBBLE_CODE`.
114
+
115
+ ## Related agents / repos
116
+
117
+ - The iPhone app has its OWN agent (separate session); communicate via
118
+ the user. App-side asks so far honored: `Pose.discontinuity` (v1.1),
119
+ sceneDepth-in-AR (v1.1), manual exposure in AR (v1.2 planned),
120
+ stabilization forced off during odometry (pending).
121
+ - `ryanrudes/irtsp-python`, `ryanrudes/irtsp-support` (wire contract).
@@ -0,0 +1 @@
1
+ AGENTS.md
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Ryan Rudes
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,283 @@
1
+ Metadata-Version: 2.4
2
+ Name: framelock
3
+ Version: 0.3.0
4
+ Summary: Synchronized multi-camera recording for multi-view pose estimation: OAK-D + iPhone RTSP + mic on one sub-ms clock, with rig registration, corresponded exports, and 3D playback
5
+ Project-URL: Repository, https://github.com/ryanrudes/framelock
6
+ Project-URL: Documentation, https://github.com/ryanrudes/framelock/tree/main/docs
7
+ Author-email: Ryan Rudes <ryanrudes@gmail.com>
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: arkit,aruco,calibration,camera,depthai,multi-view,oak-d,odometry,pose-estimation,registration,rtsp,synchronization
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Operating System :: MacOS
14
+ Classifier: Operating System :: POSIX :: Linux
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Topic :: Multimedia :: Video :: Capture
18
+ Classifier: Topic :: Scientific/Engineering :: Image Processing
19
+ Requires-Python: >=3.12
20
+ Requires-Dist: av>=18.0.0
21
+ Requires-Dist: depthai>=3.7.1
22
+ Requires-Dist: irtsp>=0.2.0
23
+ Requires-Dist: numpy>=2.5.1
24
+ Requires-Dist: opencv-python>=5.0.0.93
25
+ Requires-Dist: rich>=13.3.3
26
+ Requires-Dist: sounddevice>=0.5.5
27
+ Provides-Extra: location
28
+ Requires-Dist: pyobjc-framework-corelocation>=10.0; (sys_platform == 'darwin') and extra == 'location'
29
+ Provides-Extra: viz
30
+ Requires-Dist: viser>=1.0; extra == 'viz'
31
+ Description-Content-Type: text/markdown
32
+
33
+ # framelock
34
+
35
+ [![CI](https://github.com/ryanrudes/framelock/actions/workflows/ci.yml/badge.svg)](https://github.com/ryanrudes/framelock/actions/workflows/ci.yml)
36
+ [![PyPI](https://img.shields.io/pypi/v/framelock)](https://pypi.org/project/framelock/)
37
+ [![Python](https://img.shields.io/badge/python-3.12%2B-blue)](https://pypi.org/project/framelock/)
38
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
39
+
40
+ **Synchronized multi-camera recording for multi-view pose estimation.**
41
+ Point an OAK-D, a handful of iPhones, and a microphone at the same
42
+ scene; framelock records everything into one session folder on one
43
+ sub-millisecond shared clock, solves where every camera *is* in a
44
+ single world frame, and hands you corresponded, GVHMR-ready exports.
45
+
46
+ - **One clock.** Hardware-locked shutters inside an OAK, a verified
47
+ re-implementation of DepthAI's sync across devices, acoustic
48
+ chirp calibration for phones — every sample from every sensor lands
49
+ on one host-monotonic timeline, and every claim is cross-checked by
50
+ an independent estimator before it's trusted.
51
+ - **One world frame.** Press `b` mid-take: the laptop shows a fiducial
52
+ board with a live per-camera status strip, and the whole rig —
53
+ static cameras, phones via ARKit odometry, all three OAK cameras
54
+ fused through factory extrinsics — is solved automatically at
55
+ session end (~1 cm / 1° per camera, gravity-leveled). Printed wall
56
+ tags extend it anywhere the board can't reach.
57
+ - **Beyond pixels.** iRTSP phones stream ARKit pose, IMU, GPS,
58
+ compass, barometer, and LiDAR *on the video's own clock* (no offset
59
+ estimation — the app's clock design makes it exact); OAK-Ds
60
+ contribute IMU, stereo depth, feature tracks, and thermals.
61
+ - **Verified, not hoped.** Gyro-vs-pixels sync checks, gravity
62
+ cross-checks between accelerometers and ARKit, stereo-metric size
63
+ votes, pose-noise grading — `--verify` tells you what the take is
64
+ actually worth.
65
+
66
+ ```bash
67
+ pip install framelock # or: uv add framelock
68
+ pip install 'framelock[viz]' # + 3D playback (viser)
69
+ ```
70
+
71
+ One CLI, sensible subcommands — `framelock help` lists them:
72
+
73
+ | | |
74
+ |---|---|
75
+ | `framelock` / `framelock record` | record a session (bare flags work: `framelock --shutter-sync ...`) |
76
+ | `framelock register` | solve rig registration (board wave / tags / acoustic) |
77
+ | `framelock tags` | export printable wall fiducials at true physical scale |
78
+ | `framelock correspond` | corresponded exports + end-to-end verification |
79
+ | `framelock align` | acoustic label-bias / clock-drift measurement |
80
+ | `framelock preview` | synchronized side-by-side playback with transport |
81
+ | `framelock viz` | fly through the take in 3D (viser) |
82
+ | `framelock report` | session health report |
83
+
84
+ Beyond video: iRTSP phones stream odometry (fused IMU, ARKit 6-DOF pose,
85
+ GPS, compass, barometer, intrinsics, LiDAR depth) and framelock records it
86
+ on the **same timeline as the frames** — no offset estimation, the app's
87
+ clock design makes it exact (see `docs/OUTPUTS.md`). OAK-Ds contribute
88
+ their own IMU, stereo depth, feature tracks, and thermals via
89
+ `OakD(data=("imu", "depth", ...))`. Everything lands in the corresponded
90
+ export together, `framelock correspond --verify` checks the odometry
91
+ against the pixels, and `framelock viz` flies you through the take in 3D.
92
+
93
+ The whole rig is **registered into one world frame**
94
+ (`registration.json`). The primary ritual costs one keypress: press
95
+ `b` during a take and the laptop shows a fiducial board fullscreen
96
+ with a live per-camera status strip; show it to each camera (the
97
+ solver verifies the laptop's stillness from the data) and the rig is
98
+ solved automatically at session end — every camera ~1 cm / 1°, phones
99
+ via their ARKit odometry with timed drift anchors, all OAK cameras
100
+ fused through factory extrinsics, gravity-leveled z-up world (or pick
101
+ your own origin: `world_origin="board"` / any device / its Source
102
+ object). Board size is cross-voted by independent estimators (ruler >
103
+ stereo triangulation / ARKit motion > screen EDID), and every claim is
104
+ verified by an independent witness — gravity cross-checks between the
105
+ OAK accelerometer and each phone's ARKit measure the solved rotation's
106
+ error board-free. Printed **wall tags** ([docs/TAGS.md](docs/TAGS.md))
107
+ extend it: `framelock tags grid 0-11 --cols 4 --mm 40`, pin anywhere
108
+ fixed, and localized devices survey them automatically — every later
109
+ sighting pins phone drift or rescues a camera the board never reached.
110
+ The acoustic stop-and-chirp ritual (`g`) remains as a
111
+ no-line-of-sight fallback. Registered devices share a metric world
112
+ cell in the viz, with the board and tags drawn at their solved poses.
113
+
114
+ ```bash
115
+ # The classic single-OAK setup (all flags from the original recorder):
116
+ uv run framelock --shutter-sync --exposure 8000 --iso 800 --warmup auto -n take1
117
+
118
+ # Add an iPhone (any RTSP-server camera app):
119
+ uv run framelock --shutter-sync --exposure 8000 --iso 800 \
120
+ --rtsp iphone=rtsp://192.168.1.23:8554/live
121
+
122
+ # Digital zoom ramp 1x -> 2x between t=5s and t=10s of the recording:
123
+ uv run framelock --zoom-ramp 5:10:1.0:2.0
124
+ ```
125
+
126
+ The CLI is a thin veneer — everything is a first-class Python API
127
+ (see **[examples/](examples/)** for runnable scripts covering each feature):
128
+
129
+ ```python
130
+ from framelock import OakD, RtspCamera, Session, Warmup, ZoomRamp
131
+
132
+ oak = OakD(shutter_sync=True, shutter=1/125, iso=800,
133
+ automations=[ZoomRamp(1.0, 2.0, start=5, end=10)])
134
+ phone = RtspCamera("rtsp://192.168.1.23:8554/live", name="iphone")
135
+
136
+ with Session("recordings", name="take1", warmup=Warmup.AUTO) as session:
137
+ session.record(oak, phone)
138
+ ```
139
+
140
+ Options are typed `StrEnum`s (`VideoFormat.MP4`, `Transport.TCP`,
141
+ `ExternalSync.MASTER`, `Control.ZOOM`, `ChirpKind.VELVET`, `Warmup.AUTO`) —
142
+ IDE-completable and validated, while plain strings keep working everywhere.
143
+ Dynamic control is first-class too: `Func("zoom", lambda t: ...)` drives a
144
+ control with any function of time, and `session.record(..., on_tick=...)`
145
+ runs arbitrary per-tick logic — `apply()` is rate-safe (per-control
146
+ resolution filtering built into every source), so both can emit values
147
+ naively (see `examples/04_live_control.py`).
148
+
149
+ Previews are optional (`ui=False`, or `preview=False` per source) and render
150
+ as one CCTV-style composite window — labeled tiles, REC indicator, "no
151
+ signal" placeholders. Start/stop is console-driven: `warmup=Warmup.MANUAL`
152
+ streams and settles until **you** press `r` in the terminal; `q` stops
153
+ (with the end chirp when `end_chirp=True`); `session.request_start()` /
154
+ `request_stop()` are the programmatic equivalents (thread-safe).
155
+
156
+ ## Output layout
157
+
158
+ ```
159
+ recordings/take1/
160
+ session.json manifest: sources, clock map, settings, accuracy notes
161
+ sync_groups.csv cross-device alignment (per group: each stream's seq + ts)
162
+ registration.json every device's pose in ONE world frame (+ board, tags)
163
+ audio_alignment.json measured per-source label bias / clock drift
164
+ corresponded/ equal-frame-count videos + pose-per-frame exports
165
+ oak/
166
+ CAM_A.mp4 CAM_A.mcap CAM_A.intrinsics.json CAM_A.timestamps.csv
167
+ CAM_B.* CAM_C.* calibration.json
168
+ depth.mkv features.csv imu_*.csv system.csv with data=(...)
169
+ iphone/
170
+ video.mp4 video.timestamps.csv
171
+ imu.csv pose.csv gnss.csv ... when the iRTSP app streams them
172
+ ```
173
+
174
+ Every video has a `*.timestamps.csv` (file frame index ↔ sequence number ↔
175
+ host-monotonic + epoch time) and `sync_groups.csv` joins them across devices,
176
+ so post-hoc alignment never depends on the live estimates. Full column
177
+ reference for every file: **[docs/OUTPUTS.md](docs/OUTPUTS.md)**.
178
+
179
+ ```bash
180
+ uv run framelock viz recordings/take1 # fly through the take in 3D:
181
+ # camera frusta with their images, phone trajectories from ARKit odometry,
182
+ # play/pause/scrub/step/speed, click any camera for its full details
183
+ # (needs the viz extra: uv sync --extra viz)
184
+ ```
185
+
186
+ ## Synchronization: what you actually get
187
+
188
+ | Link | Mechanism | Accuracy |
189
+ |---|---|---|
190
+ | OAK mono pair (one device) | hardware FSIN (`shutter_sync=True`) | ~0.01 ms (shutter-locked) |
191
+ | OAK color vs monos | shared clock + locked exposure | sub-ms after a few s settling |
192
+ | Two OAK devices, one host | dai XLink timesync → host clock | timestamps <1 ms comparable; shutters free-run (±½ frame) unless FSIN-wired (PoE M8 / FFC hardware only — **not** the USB OAK-D) |
193
+ | iPhone / RTSP | software only (no hw path exists into a phone) | live: tens of ms (arrival-anchored); post-hoc with sidecars + clap calibration: ~1 frame |
194
+
195
+ The host-side `Synchronizer` re-implements DepthAI's `dai.node.Sync`
196
+ algorithm (verified against depthai-core v3.7.1 source) for any number of
197
+ streams, with one deliberate improvement: nearest-match grouping instead of
198
+ first-fit, so same-rate streams can't lock into a stable one-frame-off
199
+ pairing.
200
+
201
+ ## Camera controls & automations
202
+
203
+ `OakD` accepts initial `focus`/`exposure` at construction (`zoom=True`
204
+ enables the zoom chain, which starts at 1x), live control via
205
+ `oak.apply(zoom=1.5)`, or a timeline:
206
+
207
+ ```python
208
+ from framelock import FocusStep, ZoomRamp
209
+ OakD(automations=[
210
+ ZoomRamp(1.0, 2.0, start=5, end=10), # smooth digital zoom
211
+ FocusStep([(0, 120), (8, 200)]), # refocus at t=8s
212
+ ])
213
+ ```
214
+
215
+ Every built-in control has all three shapes — `ZoomRamp`/`ZoomStep`/`ZoomFunc`,
216
+ `FocusRamp`/`FocusStep`/`FocusFunc`, `ShutterRamp`/`ShutterStep`/`ShutterFunc`,
217
+ `IsoRamp`/`IsoStep`/`IsoFunc` — as conveniences over the generic
218
+ `Ramp(control, ...)` / `Step(control, ...)` / `Func(control, fn)` /
219
+ `Automation(control, keyframes, mode)`, which drive any control a source
220
+ supports.
221
+
222
+ Digital zoom is a runtime ImageManip center-crop on the color camera
223
+ (output size constant, H.264-safe). The intrinsics sidecar automatically
224
+ switches to **per-frame** `fx/fy/cx/cy` arrays that track the zoom exactly;
225
+ distortion coefficients are crop-invariant and stay valid throughout.
226
+
227
+ ## Automatic offset calibration (audio)
228
+
229
+ The laptop's microphone doubles as an acoustic sync reference: record it as
230
+ a source (a wideband marker plays automatically at recording start — a
231
+ pleasant pseudo-random "tss" by default; --chirp-tone mls|velvet|sweep), then
232
+ cross-correlate it against the phone's audio track — any shared sound works
233
+ (the chirp, a clap, speech). Yields the phone's timeline offset at ~1 ms
234
+ precision plus its clock drift:
235
+
236
+ ```bash
237
+ uv run framelock --rtsp iphone=rtsp://... --mic -n take1
238
+ uv run framelock align recordings/take1
239
+ # label bias +43.10 ms (drift +3.2 ppm ...) -> RtspCamera(offset=-0.0431)
240
+ ```
241
+
242
+ Sound is slow (~2.9 ms/m): if the sound source is at different distances
243
+ from the phone and the laptop, pass `--distance-phone` / `--distance-mic`
244
+ (meters) and optionally `--temperature` (°C, refines the speed of sound) and
245
+ the propagation difference is corrected. `--chirp-period N` repeats the
246
+ chirp for drift markers across long, quiet takes. Audition all the marker
247
+ sounds in **[docs/chirps/](docs/chirps/)** (one .wav per tone) before choosing
248
+ via `--chirp-tone` / `Microphone(chirp=...)`.
249
+
250
+ ## Going deeper
251
+
252
+ - **[docs/SYNCHRONIZATION.md](docs/SYNCHRONIZATION.md)** — the full sync
253
+ model: clock domains (including the macOS steady_clock trap), hardware
254
+ vs software tiers with measured accuracies, the Synchronizer algorithm,
255
+ and the phone-calibration workflow.
256
+ - **[docs/OUTPUTS.md](docs/OUTPUTS.md)** — every output file and column:
257
+ sidecars, manifests, and how to join them for offline alignment.
258
+ - **[docs/TAGS.md](docs/TAGS.md)** — printed wall fiducials: export,
259
+ print at true scale, affix, and let the rig survey them.
260
+ - **[docs/CAMERA_METADATA.md](docs/CAMERA_METADATA.md)** — the GVHMR
261
+ intrinsics-sidecar spec the OAK sidecars target.
262
+ - **[examples/](examples/)** — runnable scripts: single OAK, OAK+iPhone
263
+ with acoustic calibration, zoom automations, live `on_tick` control,
264
+ custom sync chirps, offline analysis, mobile rigs (end chirp),
265
+ air conditions / speed-of-sound models (Gavioso 2025 vs Cramer 1993),
266
+ and multi-iPhone rigs (per-phone alignment + pixel-level verify).
267
+ - **[docs/MEASUREMENTS.md](docs/MEASUREMENTS.md)** /
268
+ **[docs/CHIRP_BENCHMARK.md](docs/CHIRP_BENCHMARK.md)** — reference-rig
269
+ measurement snapshots; regenerate for any session or machine with
270
+ `framelock report session <dir>` / `framelock report chirps --live`.
271
+ - `framelock --help` / `framelock align --help` /
272
+ `framelock preview --help` / `framelock report --help` — full CLI
273
+ reference.
274
+
275
+ ## Development
276
+
277
+ ```bash
278
+ uv sync # deps (+ dev group: pytest, ruff, pre-commit)
279
+ uv run pytest # full suite (~70 s, synthetic-physics GT tests)
280
+ uv run ruff check . # lint (CI enforces both)
281
+ uv run pre-commit install # once per clone: lint on every commit
282
+ uv run framelock preview # side-by-side playback of the newest session
283
+ ```