framelock 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- framelock-0.3.0/.github/workflows/ci.yml +21 -0
- framelock-0.3.0/.github/workflows/release.yml +22 -0
- framelock-0.3.0/.gitignore +17 -0
- framelock-0.3.0/.pre-commit-config.yaml +13 -0
- framelock-0.3.0/.python-version +1 -0
- framelock-0.3.0/AGENTS.md +121 -0
- framelock-0.3.0/CLAUDE.md +1 -0
- framelock-0.3.0/LICENSE +21 -0
- framelock-0.3.0/PKG-INFO +283 -0
- framelock-0.3.0/README.md +251 -0
- framelock-0.3.0/docs/CAMERA_METADATA.md +215 -0
- framelock-0.3.0/docs/CHIRP_BENCHMARK.md +34 -0
- framelock-0.3.0/docs/MEASUREMENTS.md +77 -0
- framelock-0.3.0/docs/OUTPUTS.md +337 -0
- framelock-0.3.0/docs/SYNCHRONIZATION.md +220 -0
- framelock-0.3.0/docs/TAGS.md +129 -0
- framelock-0.3.0/docs/chirps/README.md +34 -0
- framelock-0.3.0/docs/chirps/bubble_code.wav +0 -0
- framelock-0.3.0/docs/chirps/glass_plink.wav +0 -0
- framelock-0.3.0/docs/chirps/hidden_water.wav +0 -0
- framelock-0.3.0/docs/chirps/mls.wav +0 -0
- framelock-0.3.0/docs/chirps/natural_water.wav +0 -0
- framelock-0.3.0/docs/chirps/reference.wav +0 -0
- framelock-0.3.0/docs/chirps/sparkle.wav +0 -0
- framelock-0.3.0/docs/chirps/sweep.wav +0 -0
- framelock-0.3.0/docs/chirps/velvet.wav +0 -0
- framelock-0.3.0/examples/01_single_oak.py +22 -0
- framelock-0.3.0/examples/02_oak_plus_iphone.py +69 -0
- framelock-0.3.0/examples/03_zoom_automation.py +25 -0
- framelock-0.3.0/examples/04_live_control.py +42 -0
- framelock-0.3.0/examples/05_custom_chirp.py +38 -0
- framelock-0.3.0/examples/06_offline_analysis.py +51 -0
- framelock-0.3.0/examples/07_mobile_rig.py +88 -0
- framelock-0.3.0/examples/08_air_conditions.py +65 -0
- framelock-0.3.0/examples/09_multi_iphone.py +78 -0
- framelock-0.3.0/examples/10_odometry.py +114 -0
- framelock-0.3.0/examples/11_wall_tags.py +91 -0
- framelock-0.3.0/framelock/__init__.py +115 -0
- framelock-0.3.0/framelock/__main__.py +6 -0
- framelock-0.3.0/framelock/_compat.py +49 -0
- framelock-0.3.0/framelock/assets/chirps/00_reference_linear_ramp_chirp.wav +0 -0
- framelock-0.3.0/framelock/assets/chirps/01_hidden_chirp_water_ringtone.wav +0 -0
- framelock-0.3.0/framelock/assets/chirps/02_glass_plink_hidden_probe.wav +0 -0
- framelock-0.3.0/framelock/assets/chirps/03_bubble_code_microchirps.wav +0 -0
- framelock-0.3.0/framelock/assets/chirps/04_tiny_sparkle_notification_good_sync.wav +0 -0
- framelock-0.3.0/framelock/assets/chirps/05_natural_water_wood_phrase_no_obvious_ramp.wav +0 -0
- framelock-0.3.0/framelock/audioalign.py +1068 -0
- framelock-0.3.0/framelock/chirps.py +679 -0
- framelock-0.3.0/framelock/cli.py +349 -0
- framelock-0.3.0/framelock/clock.py +53 -0
- framelock-0.3.0/framelock/controls.py +247 -0
- framelock-0.3.0/framelock/correspondence.py +765 -0
- framelock-0.3.0/framelock/diagnostics.py +643 -0
- framelock-0.3.0/framelock/odometry.py +374 -0
- framelock-0.3.0/framelock/preview.py +591 -0
- framelock-0.3.0/framelock/registration/__init__.py +183 -0
- framelock-0.3.0/framelock/registration/_se3.py +81 -0
- framelock-0.3.0/framelock/registration/acoustic.py +517 -0
- framelock-0.3.0/framelock/registration/board.py +571 -0
- framelock-0.3.0/framelock/registration/cli.py +268 -0
- framelock-0.3.0/framelock/registration/store.py +252 -0
- framelock-0.3.0/framelock/registration/tagcli.py +127 -0
- framelock-0.3.0/framelock/registration/tags.py +793 -0
- framelock-0.3.0/framelock/registration/visual.py +171 -0
- framelock-0.3.0/framelock/registration/wave.py +1242 -0
- framelock-0.3.0/framelock/session.py +1781 -0
- framelock-0.3.0/framelock/soundspeed.py +370 -0
- framelock-0.3.0/framelock/sources/__init__.py +6 -0
- framelock-0.3.0/framelock/sources/base.py +177 -0
- framelock-0.3.0/framelock/sources/mic.py +356 -0
- framelock-0.3.0/framelock/sources/oak.py +1520 -0
- framelock-0.3.0/framelock/sources/phonedata.py +559 -0
- framelock-0.3.0/framelock/sources/rtsp.py +778 -0
- framelock-0.3.0/framelock/sync.py +242 -0
- framelock-0.3.0/framelock/viz.py +931 -0
- framelock-0.3.0/pyproject.toml +92 -0
- framelock-0.3.0/tests/test_audioalign.py +580 -0
- framelock-0.3.0/tests/test_chirps.py +106 -0
- framelock-0.3.0/tests/test_controls.py +55 -0
- framelock-0.3.0/tests/test_correspondence.py +565 -0
- framelock-0.3.0/tests/test_oakdata.py +183 -0
- framelock-0.3.0/tests/test_odometry.py +286 -0
- framelock-0.3.0/tests/test_phonedata.py +360 -0
- framelock-0.3.0/tests/test_preview.py +247 -0
- framelock-0.3.0/tests/test_registration.py +1392 -0
- framelock-0.3.0/tests/test_session.py +1025 -0
- framelock-0.3.0/tests/test_soundspeed.py +112 -0
- framelock-0.3.0/tests/test_sync.py +217 -0
- framelock-0.3.0/tests/test_viz.py +205 -0
- framelock-0.3.0/uv.lock +982 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v4
|
|
13
|
+
- uses: astral-sh/setup-uv@v5
|
|
14
|
+
with:
|
|
15
|
+
python-version: "3.12"
|
|
16
|
+
- name: System libraries (PortAudio for sounddevice, GL for OpenCV)
|
|
17
|
+
run: sudo apt-get update && sudo apt-get install -y libportaudio2 libgl1
|
|
18
|
+
- name: Lint
|
|
19
|
+
run: uv run ruff check .
|
|
20
|
+
- name: Tests
|
|
21
|
+
run: uv run pytest -q
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
name: release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
pypi:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
environment: pypi
|
|
11
|
+
permissions:
|
|
12
|
+
contents: read # an explicit permissions block resets the rest
|
|
13
|
+
id-token: write # PyPI Trusted Publishing (OIDC) — no token stored
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
- uses: astral-sh/setup-uv@v5
|
|
17
|
+
with:
|
|
18
|
+
python-version: "3.12"
|
|
19
|
+
- name: Build
|
|
20
|
+
run: uv build
|
|
21
|
+
- name: Publish to PyPI
|
|
22
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# uv run pre-commit install (once per clone)
|
|
2
|
+
repos:
|
|
3
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
4
|
+
rev: v0.15.14
|
|
5
|
+
hooks:
|
|
6
|
+
- id: ruff-check
|
|
7
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
8
|
+
rev: v5.0.0
|
|
9
|
+
hooks:
|
|
10
|
+
- id: check-merge-conflict
|
|
11
|
+
- id: end-of-file-fixer
|
|
12
|
+
- id: trailing-whitespace
|
|
13
|
+
args: [--markdown-linebreak-ext=md]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.12
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# framelock (GitHub: ryanrudes/framelock; local folder: oakd_record)
|
|
2
|
+
|
|
3
|
+
Synchronized multi-camera recording for multi-view pose estimation
|
|
4
|
+
(GVHMR): OAK-D (DepthAI v3) + iPhones running the user's own **iRTSP**
|
|
5
|
+
app + host mic — one session folder, sub-millisecond shared clock,
|
|
6
|
+
odometry, rig registration, corresponded exports, 3D playback.
|
|
7
|
+
|
|
8
|
+
Python 3.12, `uv`. Run everything through `uv run` (pytest, the
|
|
9
|
+
`framelock <subcommand>` CLI — record/align/correspond/preview/
|
|
10
|
+
register/tags/report/viz; `framelock-<sub>` aliases remain —
|
|
11
|
+
and `examples/*.py`). Full test suite: `uv run pytest`
|
|
12
|
+
(~70 s, must stay green). File/column reference: `docs/OUTPUTS.md`.
|
|
13
|
+
|
|
14
|
+
## The one idea everything hangs on
|
|
15
|
+
|
|
16
|
+
Every sample from every device lands on ONE host-monotonic timeline,
|
|
17
|
+
and every claim gets **verified by an independent estimator** before
|
|
18
|
+
it's trusted. When adding anything, ask: what clock is this timestamp
|
|
19
|
+
on, and what cross-checks it? Never trust a single estimator when two
|
|
20
|
+
can vote (screen-size vs ARKit vs stereo; acoustic bias vs PRFT vs
|
|
21
|
+
motion-envelope; phone stillness testimony vs static-camera testimony).
|
|
22
|
+
|
|
23
|
+
## Clock model (do not break)
|
|
24
|
+
|
|
25
|
+
- Package sync domain: Python `time.monotonic()` seconds. Epoch via a
|
|
26
|
+
per-session frozen `ClockMap`.
|
|
27
|
+
- OAK: `getTimestamp() − dai_offset`; dai uses C++ steady_clock which
|
|
28
|
+
DIVERGES from time.monotonic() on macOS by accumulated sleep (days!)
|
|
29
|
+
— the bracketed-median offset in `oak.py` is mandatory.
|
|
30
|
+
- Phone video: `ts = pts + min(arrival − pts) + offset`; acoustic label
|
|
31
|
+
bias measured per take (`audio_alignment.json`), applied at export.
|
|
32
|
+
- Phone odometry: the iRTSP app stamps everything from ONE anchor per
|
|
33
|
+
session; `unix_ts` == the video's RTCP-SR (PRFT) axis bit-for-bit, so
|
|
34
|
+
odometry → pts axis is a measured constant (`wall_to_pts`), never an
|
|
35
|
+
estimate. Contract: `irtsp-support/INTEGRATION.md`;
|
|
36
|
+
client: `ryanrudes/irtsp-python` (>=0.2.0, `Pose.discontinuity`).
|
|
37
|
+
|
|
38
|
+
## Layout
|
|
39
|
+
|
|
40
|
+
- `framelock/session.py` — record loop, keys (r/q/c/b/g), auto
|
|
41
|
+
align/register/export at session end.
|
|
42
|
+
- `framelock/sources/` — `base.py` (Source protocol), `oak.py`
|
|
43
|
+
(+`data=("imu","depth","features","system","vio")`), `rtsp.py` +
|
|
44
|
+
`phonedata.py` (iRTSP odometry/depth recorder), `mic.py` (chirps,
|
|
45
|
+
`chirps.csv`, `chirp_template.npz`).
|
|
46
|
+
- `framelock/registration/` — a package, one module per concern:
|
|
47
|
+
`_se3.py` (rotation helpers + frame constants), `board.py` (specs,
|
|
48
|
+
rendering, detection, the live 'b' display, `board_display.csv`),
|
|
49
|
+
`wave.py` (the whole-rig wave solver: continuous-time co-sightings OR
|
|
50
|
+
verified-static board, timed anchors, IMU stillness witness, gravity
|
|
51
|
+
leveling, size votes, world-origin rebasing), `tags.py` (printed wall
|
|
52
|
+
tags: surveyed by localized devices, then phone drift anchors +
|
|
53
|
+
camera rescue), `acoustic.py`
|
|
54
|
+
(stop-and-chirp solver + ritual), `visual.py` (offline screen-board),
|
|
55
|
+
`store.py` (`registration.json`, `transform_at`, coverage), `cli.py`.
|
|
56
|
+
The public surface is re-exported from `framelock.registration`.
|
|
57
|
+
- `framelock/audioalign.py` (GCC-PHAT bias/drift), `correspondence.py`
|
|
58
|
+
(timelines, corresponded export, localization-gated trim),
|
|
59
|
+
`odometry.py` (pose resample, gyro-vs-flow verifier), `viz.py`
|
|
60
|
+
(viser), `chirps.py`, `_compat.py` (macOS objc-warning muffler).
|
|
61
|
+
|
|
62
|
+
## Hard-won pitfalls (each cost a live take or a review to find)
|
|
63
|
+
|
|
64
|
+
- rich auto-highlight injects version-dependent ANSI codes → every
|
|
65
|
+
Console uses `highlight=False`. viser pins `rich<15`.
|
|
66
|
+
- BNO086/RVC2 + depthai 3.7.1: `*_CALIBRATED` IMU variants emit ZERO
|
|
67
|
+
reports — use `*_RAW` (Luxonis' own examples do). Gyro ≤400 Hz. And a
|
|
68
|
+
second `enableIMUSensor` call CLOBBERS the first (measured twice) —
|
|
69
|
+
configure ALL report types in ONE call. RAW reports live in the IMU
|
|
70
|
+
CHIP frame: on the OAK-D the BNO086 sits rotated 180° about camera z
|
|
71
|
+
(measured via solved-pose × parked-accel; assuming identity leveled
|
|
72
|
+
the world 171° upside down). `_R_CAM_FROM_IMU` in
|
|
73
|
+
`registration/wave.py`; a device-recorded `IMU_to_CAM_A` in
|
|
74
|
+
calibration.json overrides it.
|
|
75
|
+
- OAK monos get BLINDED by a bright screen at locked exposure (34%
|
|
76
|
+
saturation measured) — the board console shows it live; dim screen.
|
|
77
|
+
- Frame timestamps are END-of-exposure (`exposure_us` column recorded
|
|
78
|
+
for mid-exposure correction). RVC2 encoder ceiling ~60 fps total, but
|
|
79
|
+
the ISP-pass budget bites FIRST: every extra requestOutput shaves fps
|
|
80
|
+
off every stream at once (previews cost 3.3, depth 1.5 — measured;
|
|
81
|
+
see docs/SYNCHRONIZATION.md). Previews now decode host-side from the
|
|
82
|
+
recording taps (free); features ride the stereo-left stream; request
|
|
83
|
+
the warning's prescribed sustainable fps for GAPLESS capture. Mixed
|
|
84
|
+
per-camera output rates stall the camera. The exporter pairs
|
|
85
|
+
multi-stream sources to joint instants so drops never tear a device.
|
|
86
|
+
- Planar PnP: use IPPE + ambiguity rejection; near-frontal + far views
|
|
87
|
+
of a small board are degenerate (weights + 80 px gate handle it).
|
|
88
|
+
- ARKit odom DRIFTS silently when feature-starved (bright screen in
|
|
89
|
+
view; 30% LIMITED tracking measured) — no discontinuity event fires
|
|
90
|
+
for drift. Static-camera testimony outranks phone testimony; per-spot
|
|
91
|
+
anchors absorb wander. Cameras without odometry must sight the board
|
|
92
|
+
from their FINAL pose (their IMU gates this when present).
|
|
93
|
+
- Screen-derived board size lies ~15% on notched/scaled Macs — seed
|
|
94
|
+
only; authority order: ruler > stereo/ARKit votes > screen.
|
|
95
|
+
- `generateImage` carves margins from the requested size (use
|
|
96
|
+
`board_image()`); ChArUco `square_mm` = chessboard square edge,
|
|
97
|
+
aruco_grid `square_mm` = marker edge.
|
|
98
|
+
- cv2's key queue is process-global; keys are routed from all windows.
|
|
99
|
+
- PyAV RTSP: never use its `timeout=` interrupt callback (GIL
|
|
100
|
+
starvation drops the stream); libav `timeout` option only.
|
|
101
|
+
- Same-template chirp search windows must stay disjoint
|
|
102
|
+
(`RegistrationRitual.GAP_S` ↔ `_TOA_SEARCH_*`).
|
|
103
|
+
|
|
104
|
+
## Conventions
|
|
105
|
+
|
|
106
|
+
- ARKit: camera = `ARCamera.transform`, gravity world +Y up;
|
|
107
|
+
`R_cv = R_arkit @ diag(1,−1,−1)`; registration frames z-up
|
|
108
|
+
(`R_WORLD_FROM_ODOM` in `registration.py` is the single source).
|
|
109
|
+
- dai extrinsics are centimeters — `calibration.json` stores meters.
|
|
110
|
+
- Tests are synthetic-physics: build takes from ground truth, assert
|
|
111
|
+
recovery (see `tests/test_registration.py` `_wave_rig`,
|
|
112
|
+
`_write_synthetic_take`). New solvers need a GT-recovery test.
|
|
113
|
+
- Live testing: user prefers `ChirpKind.BUBBLE_CODE`.
|
|
114
|
+
|
|
115
|
+
## Related agents / repos
|
|
116
|
+
|
|
117
|
+
- The iPhone app has its OWN agent (separate session); communicate via
|
|
118
|
+
the user. App-side asks so far honored: `Pose.discontinuity` (v1.1),
|
|
119
|
+
sceneDepth-in-AR (v1.1), manual exposure in AR (v1.2 planned),
|
|
120
|
+
stabilization forced off during odometry (pending).
|
|
121
|
+
- `ryanrudes/irtsp-python`, `ryanrudes/irtsp-support` (wire contract).
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
AGENTS.md
|
framelock-0.3.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ryan Rudes
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
framelock-0.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: framelock
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Synchronized multi-camera recording for multi-view pose estimation: OAK-D + iPhone RTSP + mic on one sub-ms clock, with rig registration, corresponded exports, and 3D playback
|
|
5
|
+
Project-URL: Repository, https://github.com/ryanrudes/framelock
|
|
6
|
+
Project-URL: Documentation, https://github.com/ryanrudes/framelock/tree/main/docs
|
|
7
|
+
Author-email: Ryan Rudes <ryanrudes@gmail.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: arkit,aruco,calibration,camera,depthai,multi-view,oak-d,odometry,pose-estimation,registration,rtsp,synchronization
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: MacOS
|
|
14
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Topic :: Multimedia :: Video :: Capture
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Image Processing
|
|
19
|
+
Requires-Python: >=3.12
|
|
20
|
+
Requires-Dist: av>=18.0.0
|
|
21
|
+
Requires-Dist: depthai>=3.7.1
|
|
22
|
+
Requires-Dist: irtsp>=0.2.0
|
|
23
|
+
Requires-Dist: numpy>=2.5.1
|
|
24
|
+
Requires-Dist: opencv-python>=5.0.0.93
|
|
25
|
+
Requires-Dist: rich>=13.3.3
|
|
26
|
+
Requires-Dist: sounddevice>=0.5.5
|
|
27
|
+
Provides-Extra: location
|
|
28
|
+
Requires-Dist: pyobjc-framework-corelocation>=10.0; (sys_platform == 'darwin') and extra == 'location'
|
|
29
|
+
Provides-Extra: viz
|
|
30
|
+
Requires-Dist: viser>=1.0; extra == 'viz'
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
# framelock
|
|
34
|
+
|
|
35
|
+
[](https://github.com/ryanrudes/framelock/actions/workflows/ci.yml)
|
|
36
|
+
[](https://pypi.org/project/framelock/)
|
|
37
|
+
[](https://pypi.org/project/framelock/)
|
|
38
|
+
[](LICENSE)
|
|
39
|
+
|
|
40
|
+
**Synchronized multi-camera recording for multi-view pose estimation.**
|
|
41
|
+
Point an OAK-D, a handful of iPhones, and a microphone at the same
|
|
42
|
+
scene; framelock records everything into one session folder on one
|
|
43
|
+
sub-millisecond shared clock, solves where every camera *is* in a
|
|
44
|
+
single world frame, and hands you corresponded, GVHMR-ready exports.
|
|
45
|
+
|
|
46
|
+
- **One clock.** Hardware-locked shutters inside an OAK, a verified
|
|
47
|
+
re-implementation of DepthAI's sync across devices, acoustic
|
|
48
|
+
chirp calibration for phones — every sample from every sensor lands
|
|
49
|
+
on one host-monotonic timeline, and every claim is cross-checked by
|
|
50
|
+
an independent estimator before it's trusted.
|
|
51
|
+
- **One world frame.** Press `b` mid-take: the laptop shows a fiducial
|
|
52
|
+
board with a live per-camera status strip, and the whole rig —
|
|
53
|
+
static cameras, phones via ARKit odometry, all three OAK cameras
|
|
54
|
+
fused through factory extrinsics — is solved automatically at
|
|
55
|
+
session end (~1 cm / 1° per camera, gravity-leveled). Printed wall
|
|
56
|
+
tags extend it anywhere the board can't reach.
|
|
57
|
+
- **Beyond pixels.** iRTSP phones stream ARKit pose, IMU, GPS,
|
|
58
|
+
compass, barometer, and LiDAR *on the video's own clock* (no offset
|
|
59
|
+
estimation — the app's clock design makes it exact); OAK-Ds
|
|
60
|
+
contribute IMU, stereo depth, feature tracks, and thermals.
|
|
61
|
+
- **Verified, not hoped.** Gyro-vs-pixels sync checks, gravity
|
|
62
|
+
cross-checks between accelerometers and ARKit, stereo-metric size
|
|
63
|
+
votes, pose-noise grading — `--verify` tells you what the take is
|
|
64
|
+
actually worth.
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
pip install framelock # or: uv add framelock
|
|
68
|
+
pip install 'framelock[viz]' # + 3D playback (viser)
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
One CLI, sensible subcommands — `framelock help` lists them:
|
|
72
|
+
|
|
73
|
+
| | |
|
|
74
|
+
|---|---|
|
|
75
|
+
| `framelock` / `framelock record` | record a session (bare flags work: `framelock --shutter-sync ...`) |
|
|
76
|
+
| `framelock register` | solve rig registration (board wave / tags / acoustic) |
|
|
77
|
+
| `framelock tags` | export printable wall fiducials at true physical scale |
|
|
78
|
+
| `framelock correspond` | corresponded exports + end-to-end verification |
|
|
79
|
+
| `framelock align` | acoustic label-bias / clock-drift measurement |
|
|
80
|
+
| `framelock preview` | synchronized side-by-side playback with transport |
|
|
81
|
+
| `framelock viz` | fly through the take in 3D (viser) |
|
|
82
|
+
| `framelock report` | session health report |
|
|
83
|
+
|
|
84
|
+
Beyond video: iRTSP phones stream odometry (fused IMU, ARKit 6-DOF pose,
|
|
85
|
+
GPS, compass, barometer, intrinsics, LiDAR depth) and framelock records it
|
|
86
|
+
on the **same timeline as the frames** — no offset estimation, the app's
|
|
87
|
+
clock design makes it exact (see `docs/OUTPUTS.md`). OAK-Ds contribute
|
|
88
|
+
their own IMU, stereo depth, feature tracks, and thermals via
|
|
89
|
+
`OakD(data=("imu", "depth", ...))`. Everything lands in the corresponded
|
|
90
|
+
export together, `framelock correspond --verify` checks the odometry
|
|
91
|
+
against the pixels, and `framelock viz` flies you through the take in 3D.
|
|
92
|
+
|
|
93
|
+
The whole rig is **registered into one world frame**
|
|
94
|
+
(`registration.json`). The primary ritual costs one keypress: press
|
|
95
|
+
`b` during a take and the laptop shows a fiducial board fullscreen
|
|
96
|
+
with a live per-camera status strip; show it to each camera (the
|
|
97
|
+
solver verifies the laptop's stillness from the data) and the rig is
|
|
98
|
+
solved automatically at session end — every camera ~1 cm / 1°, phones
|
|
99
|
+
via their ARKit odometry with timed drift anchors, all OAK cameras
|
|
100
|
+
fused through factory extrinsics, gravity-leveled z-up world (or pick
|
|
101
|
+
your own origin: `world_origin="board"` / any device / its Source
|
|
102
|
+
object). Board size is cross-voted by independent estimators (ruler >
|
|
103
|
+
stereo triangulation / ARKit motion > screen EDID), and every claim is
|
|
104
|
+
verified by an independent witness — gravity cross-checks between the
|
|
105
|
+
OAK accelerometer and each phone's ARKit measure the solved rotation's
|
|
106
|
+
error board-free. Printed **wall tags** ([docs/TAGS.md](docs/TAGS.md))
|
|
107
|
+
extend it: `framelock tags grid 0-11 --cols 4 --mm 40`, pin anywhere
|
|
108
|
+
fixed, and localized devices survey them automatically — every later
|
|
109
|
+
sighting pins phone drift or rescues a camera the board never reached.
|
|
110
|
+
The acoustic stop-and-chirp ritual (`g`) remains as a
|
|
111
|
+
no-line-of-sight fallback. Registered devices share a metric world
|
|
112
|
+
cell in the viz, with the board and tags drawn at their solved poses.
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
# The classic single-OAK setup (all flags from the original recorder):
|
|
116
|
+
uv run framelock --shutter-sync --exposure 8000 --iso 800 --warmup auto -n take1
|
|
117
|
+
|
|
118
|
+
# Add an iPhone (any RTSP-server camera app):
|
|
119
|
+
uv run framelock --shutter-sync --exposure 8000 --iso 800 \
|
|
120
|
+
--rtsp iphone=rtsp://192.168.1.23:8554/live
|
|
121
|
+
|
|
122
|
+
# Digital zoom ramp 1x -> 2x between t=5s and t=10s of the recording:
|
|
123
|
+
uv run framelock --zoom-ramp 5:10:1.0:2.0
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
The CLI is a thin veneer — everything is a first-class Python API
|
|
127
|
+
(see **[examples/](examples/)** for runnable scripts covering each feature):
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
from framelock import OakD, RtspCamera, Session, Warmup, ZoomRamp
|
|
131
|
+
|
|
132
|
+
oak = OakD(shutter_sync=True, shutter=1/125, iso=800,
|
|
133
|
+
automations=[ZoomRamp(1.0, 2.0, start=5, end=10)])
|
|
134
|
+
phone = RtspCamera("rtsp://192.168.1.23:8554/live", name="iphone")
|
|
135
|
+
|
|
136
|
+
with Session("recordings", name="take1", warmup=Warmup.AUTO) as session:
|
|
137
|
+
session.record(oak, phone)
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Options are typed `StrEnum`s (`VideoFormat.MP4`, `Transport.TCP`,
|
|
141
|
+
`ExternalSync.MASTER`, `Control.ZOOM`, `ChirpKind.VELVET`, `Warmup.AUTO`) —
|
|
142
|
+
IDE-completable and validated, while plain strings keep working everywhere.
|
|
143
|
+
Dynamic control is first-class too: `Func("zoom", lambda t: ...)` drives a
|
|
144
|
+
control with any function of time, and `session.record(..., on_tick=...)`
|
|
145
|
+
runs arbitrary per-tick logic — `apply()` is rate-safe (per-control
|
|
146
|
+
resolution filtering built into every source), so both can emit values
|
|
147
|
+
naively (see `examples/04_live_control.py`).
|
|
148
|
+
|
|
149
|
+
Previews are optional (`ui=False`, or `preview=False` per source) and render
|
|
150
|
+
as one CCTV-style composite window — labeled tiles, REC indicator, "no
|
|
151
|
+
signal" placeholders. Start/stop is console-driven: `warmup=Warmup.MANUAL`
|
|
152
|
+
streams and settles until **you** press `r` in the terminal; `q` stops
|
|
153
|
+
(with the end chirp when `end_chirp=True`); `session.request_start()` /
|
|
154
|
+
`request_stop()` are the programmatic equivalents (thread-safe).
|
|
155
|
+
|
|
156
|
+
## Output layout
|
|
157
|
+
|
|
158
|
+
```
|
|
159
|
+
recordings/take1/
|
|
160
|
+
session.json manifest: sources, clock map, settings, accuracy notes
|
|
161
|
+
sync_groups.csv cross-device alignment (per group: each stream's seq + ts)
|
|
162
|
+
registration.json every device's pose in ONE world frame (+ board, tags)
|
|
163
|
+
audio_alignment.json measured per-source label bias / clock drift
|
|
164
|
+
corresponded/ equal-frame-count videos + pose-per-frame exports
|
|
165
|
+
oak/
|
|
166
|
+
CAM_A.mp4 CAM_A.mcap CAM_A.intrinsics.json CAM_A.timestamps.csv
|
|
167
|
+
CAM_B.* CAM_C.* calibration.json
|
|
168
|
+
depth.mkv features.csv imu_*.csv system.csv with data=(...)
|
|
169
|
+
iphone/
|
|
170
|
+
video.mp4 video.timestamps.csv
|
|
171
|
+
imu.csv pose.csv gnss.csv ... when the iRTSP app streams them
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
Every video has a `*.timestamps.csv` (file frame index ↔ sequence number ↔
|
|
175
|
+
host-monotonic + epoch time) and `sync_groups.csv` joins them across devices,
|
|
176
|
+
so post-hoc alignment never depends on the live estimates. Full column
|
|
177
|
+
reference for every file: **[docs/OUTPUTS.md](docs/OUTPUTS.md)**.
|
|
178
|
+
|
|
179
|
+
```bash
|
|
180
|
+
uv run framelock viz recordings/take1 # fly through the take in 3D:
|
|
181
|
+
# camera frusta with their images, phone trajectories from ARKit odometry,
|
|
182
|
+
# play/pause/scrub/step/speed, click any camera for its full details
|
|
183
|
+
# (needs the viz extra: uv sync --extra viz)
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
## Synchronization: what you actually get
|
|
187
|
+
|
|
188
|
+
| Link | Mechanism | Accuracy |
|
|
189
|
+
|---|---|---|
|
|
190
|
+
| OAK mono pair (one device) | hardware FSIN (`shutter_sync=True`) | ~0.01 ms (shutter-locked) |
|
|
191
|
+
| OAK color vs monos | shared clock + locked exposure | sub-ms after a few s settling |
|
|
192
|
+
| Two OAK devices, one host | dai XLink timesync → host clock | timestamps <1 ms comparable; shutters free-run (±½ frame) unless FSIN-wired (PoE M8 / FFC hardware only — **not** the USB OAK-D) |
|
|
193
|
+
| iPhone / RTSP | software only (no hw path exists into a phone) | live: tens of ms (arrival-anchored); post-hoc with sidecars + clap calibration: ~1 frame |
|
|
194
|
+
|
|
195
|
+
The host-side `Synchronizer` re-implements DepthAI's `dai.node.Sync`
|
|
196
|
+
algorithm (verified against depthai-core v3.7.1 source) for any number of
|
|
197
|
+
streams, with one deliberate improvement: nearest-match grouping instead of
|
|
198
|
+
first-fit, so same-rate streams can't lock into a stable one-frame-off
|
|
199
|
+
pairing.
|
|
200
|
+
|
|
201
|
+
## Camera controls & automations
|
|
202
|
+
|
|
203
|
+
`OakD` accepts initial `focus`/`exposure` at construction (`zoom=True`
|
|
204
|
+
enables the zoom chain, which starts at 1x), live control via
|
|
205
|
+
`oak.apply(zoom=1.5)`, or a timeline:
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
from framelock import FocusStep, ZoomRamp
|
|
209
|
+
OakD(automations=[
|
|
210
|
+
ZoomRamp(1.0, 2.0, start=5, end=10), # smooth digital zoom
|
|
211
|
+
FocusStep([(0, 120), (8, 200)]), # refocus at t=8s
|
|
212
|
+
])
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
Every built-in control has all three shapes — `ZoomRamp`/`ZoomStep`/`ZoomFunc`,
|
|
216
|
+
`FocusRamp`/`FocusStep`/`FocusFunc`, `ShutterRamp`/`ShutterStep`/`ShutterFunc`,
|
|
217
|
+
`IsoRamp`/`IsoStep`/`IsoFunc` — as conveniences over the generic
|
|
218
|
+
`Ramp(control, ...)` / `Step(control, ...)` / `Func(control, fn)` /
|
|
219
|
+
`Automation(control, keyframes, mode)`, which drive any control a source
|
|
220
|
+
supports.
|
|
221
|
+
|
|
222
|
+
Digital zoom is a runtime ImageManip center-crop on the color camera
|
|
223
|
+
(output size constant, H.264-safe). The intrinsics sidecar automatically
|
|
224
|
+
switches to **per-frame** `fx/fy/cx/cy` arrays that track the zoom exactly;
|
|
225
|
+
distortion coefficients are crop-invariant and stay valid throughout.
|
|
226
|
+
|
|
227
|
+
## Automatic offset calibration (audio)
|
|
228
|
+
|
|
229
|
+
The laptop's microphone doubles as an acoustic sync reference: record it as
|
|
230
|
+
a source (a wideband marker plays automatically at recording start — a
|
|
231
|
+
pleasant pseudo-random "tss" by default; --chirp-tone mls|velvet|sweep), then
|
|
232
|
+
cross-correlate it against the phone's audio track — any shared sound works
|
|
233
|
+
(the chirp, a clap, speech). Yields the phone's timeline offset at ~1 ms
|
|
234
|
+
precision plus its clock drift:
|
|
235
|
+
|
|
236
|
+
```bash
|
|
237
|
+
uv run framelock --rtsp iphone=rtsp://... --mic -n take1
|
|
238
|
+
uv run framelock align recordings/take1
|
|
239
|
+
# label bias +43.10 ms (drift +3.2 ppm ...) -> RtspCamera(offset=-0.0431)
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
Sound is slow (~2.9 ms/m): if the sound source is at different distances
|
|
243
|
+
from the phone and the laptop, pass `--distance-phone` / `--distance-mic`
|
|
244
|
+
(meters) and optionally `--temperature` (°C, refines the speed of sound) and
|
|
245
|
+
the propagation difference is corrected. `--chirp-period N` repeats the
|
|
246
|
+
chirp for drift markers across long, quiet takes. Audition all the marker
|
|
247
|
+
sounds in **[docs/chirps/](docs/chirps/)** (one .wav per tone) before choosing
|
|
248
|
+
via `--chirp-tone` / `Microphone(chirp=...)`.
|
|
249
|
+
|
|
250
|
+
## Going deeper
|
|
251
|
+
|
|
252
|
+
- **[docs/SYNCHRONIZATION.md](docs/SYNCHRONIZATION.md)** — the full sync
|
|
253
|
+
model: clock domains (including the macOS steady_clock trap), hardware
|
|
254
|
+
vs software tiers with measured accuracies, the Synchronizer algorithm,
|
|
255
|
+
and the phone-calibration workflow.
|
|
256
|
+
- **[docs/OUTPUTS.md](docs/OUTPUTS.md)** — every output file and column:
|
|
257
|
+
sidecars, manifests, and how to join them for offline alignment.
|
|
258
|
+
- **[docs/TAGS.md](docs/TAGS.md)** — printed wall fiducials: export,
|
|
259
|
+
print at true scale, affix, and let the rig survey them.
|
|
260
|
+
- **[docs/CAMERA_METADATA.md](docs/CAMERA_METADATA.md)** — the GVHMR
|
|
261
|
+
intrinsics-sidecar spec the OAK sidecars target.
|
|
262
|
+
- **[examples/](examples/)** — runnable scripts: single OAK, OAK+iPhone
|
|
263
|
+
with acoustic calibration, zoom automations, live `on_tick` control,
|
|
264
|
+
custom sync chirps, offline analysis, mobile rigs (end chirp),
|
|
265
|
+
air conditions / speed-of-sound models (Gavioso 2025 vs Cramer 1993),
|
|
266
|
+
and multi-iPhone rigs (per-phone alignment + pixel-level verify).
|
|
267
|
+
- **[docs/MEASUREMENTS.md](docs/MEASUREMENTS.md)** /
|
|
268
|
+
**[docs/CHIRP_BENCHMARK.md](docs/CHIRP_BENCHMARK.md)** — reference-rig
|
|
269
|
+
measurement snapshots; regenerate for any session or machine with
|
|
270
|
+
`framelock report session <dir>` / `framelock report chirps --live`.
|
|
271
|
+
- `framelock --help` / `framelock align --help` /
|
|
272
|
+
`framelock preview --help` / `framelock report --help` — full CLI
|
|
273
|
+
reference.
|
|
274
|
+
|
|
275
|
+
## Development
|
|
276
|
+
|
|
277
|
+
```bash
|
|
278
|
+
uv sync # deps (+ dev group: pytest, ruff, pre-commit)
|
|
279
|
+
uv run pytest # full suite (~70 s, synthetic-physics GT tests)
|
|
280
|
+
uv run ruff check . # lint (CI enforces both)
|
|
281
|
+
uv run pre-commit install # once per clone: lint on every commit
|
|
282
|
+
uv run framelock preview # side-by-side playback of the newest session
|
|
283
|
+
```
|