worldgap 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- worldgap-0.1.0/.github/workflows/ci.yml +32 -0
- worldgap-0.1.0/.gitignore +22 -0
- worldgap-0.1.0/CHANGELOG.md +111 -0
- worldgap-0.1.0/LICENSE +21 -0
- worldgap-0.1.0/PKG-INFO +128 -0
- worldgap-0.1.0/README.md +87 -0
- worldgap-0.1.0/ROADMAP.md +104 -0
- worldgap-0.1.0/configs/v1_default.yaml +27 -0
- worldgap-0.1.0/configs/v2_default.yaml +29 -0
- worldgap-0.1.0/docs/TECHNICAL_SPEC.md +452 -0
- worldgap-0.1.0/docs/architecture.md +53 -0
- worldgap-0.1.0/docs/data_spec.md +64 -0
- worldgap-0.1.0/notebooks/demo.ipynb +584 -0
- worldgap-0.1.0/pyproject.toml +76 -0
- worldgap-0.1.0/scripts/download_datasets.sh +46 -0
- worldgap-0.1.0/src/worldgap/__init__.py +25 -0
- worldgap-0.1.0/src/worldgap/analyzer.py +212 -0
- worldgap-0.1.0/src/worldgap/cli.py +268 -0
- worldgap-0.1.0/src/worldgap/config.py +100 -0
- worldgap-0.1.0/src/worldgap/data/__init__.py +0 -0
- worldgap-0.1.0/src/worldgap/data/index.py +155 -0
- worldgap-0.1.0/src/worldgap/data/loaders/__init__.py +0 -0
- worldgap-0.1.0/src/worldgap/data/loaders/egohands.py +39 -0
- worldgap-0.1.0/src/worldgap/data/loaders/hagrid.py +82 -0
- worldgap-0.1.0/src/worldgap/data/loaders/pgm_actuator.py +110 -0
- worldgap-0.1.0/src/worldgap/data/loaders/synthetic_perturb.py +153 -0
- worldgap-0.1.0/src/worldgap/data/rollout.py +149 -0
- worldgap-0.1.0/src/worldgap/metrics/__init__.py +0 -0
- worldgap-0.1.0/src/worldgap/metrics/frechet.py +104 -0
- worldgap-0.1.0/src/worldgap/metrics/mmd.py +64 -0
- worldgap-0.1.0/src/worldgap/models/__init__.py +0 -0
- worldgap-0.1.0/src/worldgap/models/collapse.py +32 -0
- worldgap-0.1.0/src/worldgap/models/ema.py +26 -0
- worldgap-0.1.0/src/worldgap/models/encoders/__init__.py +0 -0
- worldgap-0.1.0/src/worldgap/models/encoders/actuator_encoder.py +48 -0
- worldgap-0.1.0/src/worldgap/models/encoders/common.py +50 -0
- worldgap-0.1.0/src/worldgap/models/encoders/landmark_encoder.py +48 -0
- worldgap-0.1.0/src/worldgap/models/world_model.py +96 -0
- worldgap-0.1.0/src/worldgap/report.py +228 -0
- worldgap-0.1.0/src/worldgap/validation/__init__.py +0 -0
- worldgap-0.1.0/src/worldgap/validation/harness.py +90 -0
- worldgap-0.1.0/src/worldgap/validation/stats.py +78 -0
- worldgap-0.1.0/tests/test_checkpoint.py +89 -0
- worldgap-0.1.0/tests/test_cli.py +166 -0
- worldgap-0.1.0/tests/test_collapse_safeguard.py +72 -0
- worldgap-0.1.0/tests/test_config_yaml.py +51 -0
- worldgap-0.1.0/tests/test_frechet_numerics.py +61 -0
- worldgap-0.1.0/tests/test_hagrid_loader.py +25 -0
- worldgap-0.1.0/tests/test_index.py +114 -0
- worldgap-0.1.0/tests/test_mmd.py +30 -0
- worldgap-0.1.0/tests/test_modality_swap.py +92 -0
- worldgap-0.1.0/tests/test_pgm_hysteresis.py +64 -0
- worldgap-0.1.0/tests/test_report.py +88 -0
- worldgap-0.1.0/tests/test_rollout_schema.py +81 -0
- worldgap-0.1.0/tests/test_stats.py +59 -0
- worldgap-0.1.0/tests/test_synthetic_perturb.py +121 -0
- worldgap-0.1.0/tests/test_validation_harness.py +51 -0
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
matrix:
|
|
14
|
+
python-version: ["3.10", "3.11", "3.12"]
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
|
|
18
|
+
- name: Set up Python
|
|
19
|
+
uses: actions/setup-python@v5
|
|
20
|
+
with:
|
|
21
|
+
python-version: ${{ matrix.python-version }}
|
|
22
|
+
|
|
23
|
+
- name: Install dependencies
|
|
24
|
+
run: |
|
|
25
|
+
python -m pip install --upgrade pip
|
|
26
|
+
pip install -e ".[dev]"
|
|
27
|
+
|
|
28
|
+
- name: Lint
|
|
29
|
+
run: ruff check src tests
|
|
30
|
+
|
|
31
|
+
- name: Test
|
|
32
|
+
run: pytest tests/ -v
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.pyc
|
|
3
|
+
*.egg-info/
|
|
4
|
+
.pytest_cache/
|
|
5
|
+
.ruff_cache/
|
|
6
|
+
build/
|
|
7
|
+
dist/
|
|
8
|
+
.venv/
|
|
9
|
+
venv/
|
|
10
|
+
|
|
11
|
+
# data (spec 12.16 -- raw datasets MUST NOT be committed)
|
|
12
|
+
data/raw/
|
|
13
|
+
data/processed/
|
|
14
|
+
data/index.db
|
|
15
|
+
|
|
16
|
+
# experiment tracking / notebooks scratch
|
|
17
|
+
wandb/
|
|
18
|
+
.ipynb_checkpoints/
|
|
19
|
+
notebooks/*.html
|
|
20
|
+
notebooks/*.png
|
|
21
|
+
*.pt
|
|
22
|
+
*.pth
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
4
|
+
|
|
5
|
+
## [Unreleased]
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
- `pyproject.toml`: added `[project.urls]` (Homepage/Repository/Issues/Changelog)
|
|
9
|
+
and completed the classifier list (Python 3.10/3.11/3.12, OS Independent,
|
|
10
|
+
`Human Machine Interfaces` in place of a nonexistent `Robotics` topic
|
|
11
|
+
classifier — verified against the real `trove-classifiers` package, not
|
|
12
|
+
guessed). Verified `python -m build` + `twine check` pass, and that the
|
|
13
|
+
built wheel installs and runs correctly (`import worldgap`, `GapAnalyzer`,
|
|
14
|
+
and the `worldgap` console script) in a completely clean venv.
|
|
15
|
+
- `notebooks/demo.ipynb`: added a Colab badge and an auto-install cell so the
|
|
16
|
+
notebook works from a fresh Colab runtime (installs from GitHub until the
|
|
17
|
+
package is on PyPI, then from PyPI). Re-verified zero errors after the change.
|
|
18
|
+
- `README.md`: added PyPI and Colab badges.
|
|
19
|
+
- `data/index.py`: `RolloutIndex`, a SQLite metadata index per spec 5.3.
|
|
20
|
+
`Rollout.save()`/`Rollout.load()` alone can't round-trip a rollout's
|
|
21
|
+
condition/source/metadata without the caller already knowing them
|
|
22
|
+
out-of-band; this closes that gap and is what makes "load every rollout in
|
|
23
|
+
a directory" possible. Tested in `tests/test_index.py`.
|
|
24
|
+
- `GapAnalyzer.save_checkpoint()` / `GapAnalyzer.load_checkpoint()`: model +
|
|
25
|
+
optimizer + config persistence, needed so `worldgap train` and
|
|
26
|
+
`worldgap analyze` can be separate processes. Tested in
|
|
27
|
+
`tests/test_checkpoint.py`.
|
|
28
|
+
- `GapConfig.from_yaml()`: loads `configs/v1_default.yaml`-style files for
|
|
29
|
+
`worldgap train --config`. Tested against the real default configs in
|
|
30
|
+
`tests/test_config_yaml.py`.
|
|
31
|
+
- `report.py`: HTML/Markdown report generation per spec 9.3 — condition
|
|
32
|
+
table, Frechet/MMD trend plot, low-confidence warning surfacing, and a
|
|
33
|
+
Frechet/MMD rank-disagreement diagnostic per spec Section 210. Tested in
|
|
34
|
+
`tests/test_report.py`.
|
|
35
|
+
- CLI wired end-to-end: `train`, `analyze`, and `validate` now actually call
|
|
36
|
+
`GapAnalyzer`/`ValidationHarness` against local rollout stores rather than
|
|
37
|
+
only parsing arguments. Documented decision: each of
|
|
38
|
+
`--data-dir`/`--source`/`--target` is a self-contained rollout store
|
|
39
|
+
(`{dir}/index.db` + `{dir}/{modality}/*.npz`) rather than assuming one
|
|
40
|
+
shared repo-wide `data/` root — see `cli.py`'s module docstring and the
|
|
41
|
+
corresponding note added to spec 9.2. Tested end-to-end in
|
|
42
|
+
`tests/test_cli.py`, including against the real installed console-script.
|
|
43
|
+
- `notebooks/demo.ipynb`: runs top-to-bottom via `jupyter nbconvert --execute`
|
|
44
|
+
with zero manual intervention (spec Section 13 acceptance criterion) —
|
|
45
|
+
covers V1 perception gap with a perturbation-severity sanity sweep, the
|
|
46
|
+
V1/V2 reusability claim via the identical `GapAnalyzer` class, report
|
|
47
|
+
generation, and the validation harness's anti-cherry-picking rejection.
|
|
48
|
+
Entirely synthetic data; no Kaggle/MediaPipe/GPU access needed.
|
|
49
|
+
- `matplotlib` added as a core dependency (needed by `report.py`).
|
|
50
|
+
|
|
51
|
+
### Fixed
|
|
52
|
+
- `inject_tremor` / `reduce_range_of_motion` (`data/loaders/synthetic_perturb.py`)
|
|
53
|
+
were perturbing every state channel, including the pose block's `visibility`
|
|
54
|
+
column — contradicting `inject_tremor`'s own docstring and silently injecting
|
|
55
|
+
noise into what would later be MediaPipe-confidence-derived validation
|
|
56
|
+
ground truth (spec 8.1). Both now use a new `perception_position_channel_mask`
|
|
57
|
+
(`data/rollout.py`) to restrict spatial perturbations to x/y/z channels only,
|
|
58
|
+
for real 258-dim perception rollouts; behavior on non-standard-shape
|
|
59
|
+
(test-fixture/actuation) rollouts is unchanged.
|
|
60
|
+
- `cli.py`'s module docstring claimed `train`/`analyze` were "wired to
|
|
61
|
+
GapAnalyzer" — they aren't yet (matches ROADMAP.md, which was already
|
|
62
|
+
accurate); docstring corrected to match actual behavior.
|
|
63
|
+
- `GapResult` (`analyzer.py`) now exposes `n_source`/`n_target`/`confidence`
|
|
64
|
+
as read-only properties proxying `.frechet`, matching spec 9.1's API
|
|
65
|
+
example literally rather than only via `result.frechet.n_source`.
|
|
66
|
+
- `spearman_with_bootstrap_ci` (`validation/stats.py`) now raises a clear
|
|
67
|
+
error instead of crashing on `np.percentile` if every bootstrap resample
|
|
68
|
+
produces a NaN rho (fully degenerate/constant input).
|
|
69
|
+
- `docs/TECHNICAL_SPEC.md` Sections 9, 10, 13 still referenced the project's
|
|
70
|
+
old name (`simgap`) in code samples and the repo-tree block; corrected to
|
|
71
|
+
`worldgap` throughout. Section 12.13 also gained an explicit note
|
|
72
|
+
documenting that the hysteresis fit is a two-branch polynomial split, not
|
|
73
|
+
literally Bouc-Wen/Hammerstein-Wiener as the MUST names — a conscious,
|
|
74
|
+
tested simplification rather than an unremarked deviation.
|
|
75
|
+
|
|
76
|
+
### Added
|
|
77
|
+
- Initial repo scaffolding: `pyproject.toml`, src-layout package, CI workflow.
|
|
78
|
+
- `Rollout` schema with content-hashed IDs, save/load round-trip (spec 5.1–5.3).
|
|
79
|
+
- Synthetic perturbation pipeline: tremor injection, reduced range-of-motion,
|
|
80
|
+
occlusion simulation — all seeded and reproducible (spec 5.4).
|
|
81
|
+
- `LandmarkEncoder` (Transformer, spec 6.1) and `ActuatorEncoder` (TCN, spec 6.2).
|
|
82
|
+
- Shared `WorldModel` JEPA-style core with EMA target encoder and an automated
|
|
83
|
+
representation-collapse safeguard (spec 6.3, edge case 12.7).
|
|
84
|
+
- Fréchet-distance divergence metric with Ledoit-Wolf covariance shrinkage,
|
|
85
|
+
numerically-stable matrix-sqrt handling, and a sample-size confidence flag
|
|
86
|
+
(spec 7.1, 7.3, edge case 12.9).
|
|
87
|
+
- MMD cross-check metric (spec 7.2).
|
|
88
|
+
- `GapAnalyzer` top-level API, verified identical across the perception and
|
|
89
|
+
actuation modalities via `tests/test_modality_swap.py` — the concrete test
|
|
90
|
+
of the "one core, swappable encoder" reusability claim (spec Section 3).
|
|
91
|
+
- `ValidationHarness` with pre-registration enforced in code, not just
|
|
92
|
+
documented — submitting a condition set that doesn't exactly match what was
|
|
93
|
+
pre-registered raises rather than silently proceeding (spec 8.3).
|
|
94
|
+
- Two-branch (loading/unloading) hysteresis-aware curve fit for PGM actuator
|
|
95
|
+
reference data, with a residual-structure diagnostic that catches the naive
|
|
96
|
+
single-branch mistake (spec 5.5, edge case 12.13).
|
|
97
|
+
- HaGRID/EgoHands loaders: directory scanning and canonical-gesture filtering
|
|
98
|
+
implemented and tested; MediaPipe landmark extraction left as a documented
|
|
99
|
+
`NotImplementedError` seam (needs real data + network access unavailable in
|
|
100
|
+
the scaffolding sandbox).
|
|
101
|
+
- CLI skeleton (`worldgap train/analyze/validate`).
|
|
102
|
+
- `docs/TECHNICAL_SPEC.md`, `docs/architecture.md`, `docs/data_spec.md`,
|
|
103
|
+
`ROADMAP.md`.
|
|
104
|
+
|
|
105
|
+
### Known gaps (see ROADMAP.md)
|
|
106
|
+
- No dataset actually downloaded or run through MediaPipe yet.
|
|
107
|
+
- No real PGM characterization data digitized yet (synthetic data only, for
|
|
108
|
+
testing the fitting code itself).
|
|
109
|
+
- No end-to-end V1 validation run against real ground truth yet.
|
|
110
|
+
- CLI does not yet call the loaders end-to-end.
|
|
111
|
+
- Report generation (spec 9.3) not implemented.
|
worldgap-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Min
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
worldgap-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: worldgap
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Reusable world-model-based domain-gap quantification for perception and actuation pipelines, before touching hardware.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Mattral/worldgap
|
|
6
|
+
Project-URL: Repository, https://github.com/Mattral/worldgap
|
|
7
|
+
Project-URL: Issues, https://github.com/Mattral/worldgap/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/Mattral/worldgap/blob/main/CHANGELOG.md
|
|
9
|
+
Author: Min (Mattral)
|
|
10
|
+
License: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: domain-gap,jepa,rehabilitation,robotics,sim2real,world-model
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Human Machine Interfaces
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Requires-Dist: matplotlib>=3.7
|
|
25
|
+
Requires-Dist: numpy>=1.26
|
|
26
|
+
Requires-Dist: pandas>=2.1
|
|
27
|
+
Requires-Dist: pydantic>=2.5
|
|
28
|
+
Requires-Dist: pyyaml>=6.0
|
|
29
|
+
Requires-Dist: scikit-learn>=1.3
|
|
30
|
+
Requires-Dist: scipy>=1.11
|
|
31
|
+
Requires-Dist: torch>=2.2
|
|
32
|
+
Provides-Extra: actuation
|
|
33
|
+
Requires-Dist: mujoco>=3.1; extra == 'actuation'
|
|
34
|
+
Provides-Extra: dev
|
|
35
|
+
Requires-Dist: hypothesis>=6.100; extra == 'dev'
|
|
36
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
37
|
+
Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
38
|
+
Provides-Extra: perception
|
|
39
|
+
Requires-Dist: mediapipe>=0.10; extra == 'perception'
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# worldgap
|
|
43
|
+
|
|
44
|
+
[](https://pypi.org/project/worldgap/)
|
|
45
|
+
[](https://colab.research.google.com/github/Mattral/worldgap/blob/main/notebooks/demo.ipynb)
|
|
46
|
+
|
|
47
|
+
Reusable world-model-based domain-gap quantification — for perception pipelines and
|
|
48
|
+
actuator/mechanism models — before any hardware is touched.
|
|
49
|
+
|
|
50
|
+
Ground truth for this project's design is [`docs/TECHNICAL_SPEC.md`](docs/TECHNICAL_SPEC.md).
|
|
51
|
+
Read that before changing architecture, data schemas, or metric definitions.
|
|
52
|
+
|
|
53
|
+
## What this is
|
|
54
|
+
|
|
55
|
+
Given two sets of rollouts (a source domain and a target domain — e.g. clean-lighting
|
|
56
|
+
hand-tracking data vs. occluded/low-light data, or a simulated actuator vs. a published
|
|
57
|
+
real-actuator characterization curve), `worldgap` trains a shared world model, encodes
|
|
58
|
+
both domains into a common latent space, and reports a divergence score that is designed
|
|
59
|
+
to *predict* real-world transfer degradation — validated against independently measured
|
|
60
|
+
ground truth, not asserted.
|
|
61
|
+
|
|
62
|
+
One core library. Two current use cases, distinguished only by which encoder plugs in:
|
|
63
|
+
|
|
64
|
+
- **V1 — perception gap**: MediaPipe landmark sequences. No hardware required.
|
|
65
|
+
- **V2 — actuation gap**: pneumatic gel muscle (PGM) pressure/response sequences, using
|
|
66
|
+
a published characterization curve as the "real" reference. No hardware required.
|
|
67
|
+
- **V3 — closed loop** (not started, contingent on lab access): swaps in live logged
|
|
68
|
+
telemetry from real hardware. Same core code, new data loader only — see spec Section 15.
|
|
69
|
+
|
|
70
|
+
## Status
|
|
71
|
+
|
|
72
|
+
Core library, CLI, report generation, and demo notebook are implemented and tested
|
|
73
|
+
end-to-end against synthetic/local data. Real-data phases (HaGRID/EgoHands MediaPipe
|
|
74
|
+
extraction, the real Ogawa et al. PGM curve) are blocked on external access this
|
|
75
|
+
environment doesn't have. See [`ROADMAP.md`](ROADMAP.md) for the exact phase-by-phase
|
|
76
|
+
status, and [`CHANGELOG.md`](CHANGELOG.md) for what's landed so far.
|
|
77
|
+
|
|
78
|
+
## Install
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
pip install -e . # core: torch + the World Model, works for both modalities
|
|
82
|
+
pip install -e ".[perception]" # + MediaPipe, for V1 data loading (HaGRID/EgoHands)
|
|
83
|
+
pip install -e ".[actuation]" # + MuJoCo, for V2 data loading/simulation
|
|
84
|
+
pip install -e ".[dev]" # test tooling
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## Quickstart
|
|
88
|
+
|
|
89
|
+
The library API works with any `Rollout` objects you construct yourself — the note
|
|
90
|
+
below only applies to *producing* rollouts from raw HaGRID/EgoHands data, which still
|
|
91
|
+
needs MediaPipe + real downloads (see ROADMAP Phase 0/1).
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
from worldgap import GapAnalyzer
|
|
95
|
+
from worldgap.config import GapConfig
|
|
96
|
+
|
|
97
|
+
config = GapConfig(modality="perception")
|
|
98
|
+
analyzer = GapAnalyzer(config)
|
|
99
|
+
analyzer.fit(train_rollouts)
|
|
100
|
+
result = analyzer.compute_gap(source_rollouts, target_rollouts)
|
|
101
|
+
print(result.frechet.distance, result.confidence)
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
See [`notebooks/demo.ipynb`](notebooks/demo.ipynb) for a runnable end-to-end example
|
|
105
|
+
(synthetic data, no external dependencies) covering both modalities, report
|
|
106
|
+
generation, and the validation harness.
|
|
107
|
+
|
|
108
|
+
## CLI
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
worldgap train --modality perception --data-dir ./data/processed --config configs/v1_default.yaml
|
|
112
|
+
worldgap analyze --source ./data/clean --target ./data/perturbed --modality perception --output report.html
|
|
113
|
+
worldgap validate --gap-scores results.csv --ground-truth degradation.csv
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
`--data-dir`/`--source`/`--target` are each a self-contained rollout store
|
|
117
|
+
(`{dir}/index.db` + `{dir}/{modality}/*.npz`, built with `Rollout.save()` +
|
|
118
|
+
`RolloutIndex.add()` — see `tests/test_index.py`). See `src/worldgap/cli.py`'s module
|
|
119
|
+
docstring for why this differs slightly from spec 5.3's single-shared-index diagram.
|
|
120
|
+
|
|
121
|
+
## Why not a webapp
|
|
122
|
+
|
|
123
|
+
This is infra meant to be dropped into someone else's pipeline, not a hosted service.
|
|
124
|
+
See spec Section 9 for the full API/CLI contract.
|
|
125
|
+
|
|
126
|
+
## License
|
|
127
|
+
|
|
128
|
+
MIT — see [`LICENSE`](LICENSE).
|
worldgap-0.1.0/README.md
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# worldgap
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/worldgap/)
|
|
4
|
+
[](https://colab.research.google.com/github/Mattral/worldgap/blob/main/notebooks/demo.ipynb)
|
|
5
|
+
|
|
6
|
+
Reusable world-model-based domain-gap quantification — for perception pipelines and
|
|
7
|
+
actuator/mechanism models — before any hardware is touched.
|
|
8
|
+
|
|
9
|
+
Ground truth for this project's design is [`docs/TECHNICAL_SPEC.md`](docs/TECHNICAL_SPEC.md).
|
|
10
|
+
Read that before changing architecture, data schemas, or metric definitions.
|
|
11
|
+
|
|
12
|
+
## What this is
|
|
13
|
+
|
|
14
|
+
Given two sets of rollouts (a source domain and a target domain — e.g. clean-lighting
|
|
15
|
+
hand-tracking data vs. occluded/low-light data, or a simulated actuator vs. a published
|
|
16
|
+
real-actuator characterization curve), `worldgap` trains a shared world model, encodes
|
|
17
|
+
both domains into a common latent space, and reports a divergence score that is designed
|
|
18
|
+
to *predict* real-world transfer degradation — validated against independently measured
|
|
19
|
+
ground truth, not asserted.
|
|
20
|
+
|
|
21
|
+
One core library. Two current use cases, distinguished only by which encoder plugs in:
|
|
22
|
+
|
|
23
|
+
- **V1 — perception gap**: MediaPipe landmark sequences. No hardware required.
|
|
24
|
+
- **V2 — actuation gap**: pneumatic gel muscle (PGM) pressure/response sequences, using
|
|
25
|
+
a published characterization curve as the "real" reference. No hardware required.
|
|
26
|
+
- **V3 — closed loop** (not started, contingent on lab access): swaps in live logged
|
|
27
|
+
telemetry from real hardware. Same core code, new data loader only — see spec Section 15.
|
|
28
|
+
|
|
29
|
+
## Status
|
|
30
|
+
|
|
31
|
+
Core library, CLI, report generation, and demo notebook are implemented and tested
|
|
32
|
+
end-to-end against synthetic/local data. Real-data phases (HaGRID/EgoHands MediaPipe
|
|
33
|
+
extraction, the real Ogawa et al. PGM curve) are blocked on external access this
|
|
34
|
+
environment doesn't have. See [`ROADMAP.md`](ROADMAP.md) for the exact phase-by-phase
|
|
35
|
+
status, and [`CHANGELOG.md`](CHANGELOG.md) for what's landed so far.
|
|
36
|
+
|
|
37
|
+
## Install
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
pip install -e . # core: torch + the World Model, works for both modalities
|
|
41
|
+
pip install -e ".[perception]" # + MediaPipe, for V1 data loading (HaGRID/EgoHands)
|
|
42
|
+
pip install -e ".[actuation]" # + MuJoCo, for V2 data loading/simulation
|
|
43
|
+
pip install -e ".[dev]" # test tooling
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Quickstart
|
|
47
|
+
|
|
48
|
+
The library API works with any `Rollout` objects you construct yourself — the note
|
|
49
|
+
below only applies to *producing* rollouts from raw HaGRID/EgoHands data, which still
|
|
50
|
+
needs MediaPipe + real downloads (see ROADMAP Phase 0/1).
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from worldgap import GapAnalyzer
|
|
54
|
+
from worldgap.config import GapConfig
|
|
55
|
+
|
|
56
|
+
config = GapConfig(modality="perception")
|
|
57
|
+
analyzer = GapAnalyzer(config)
|
|
58
|
+
analyzer.fit(train_rollouts)
|
|
59
|
+
result = analyzer.compute_gap(source_rollouts, target_rollouts)
|
|
60
|
+
print(result.frechet.distance, result.confidence)
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
See [`notebooks/demo.ipynb`](notebooks/demo.ipynb) for a runnable end-to-end example
|
|
64
|
+
(synthetic data, no external dependencies) covering both modalities, report
|
|
65
|
+
generation, and the validation harness.
|
|
66
|
+
|
|
67
|
+
## CLI
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
worldgap train --modality perception --data-dir ./data/processed --config configs/v1_default.yaml
|
|
71
|
+
worldgap analyze --source ./data/clean --target ./data/perturbed --modality perception --output report.html
|
|
72
|
+
worldgap validate --gap-scores results.csv --ground-truth degradation.csv
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
`--data-dir`/`--source`/`--target` are each a self-contained rollout store
|
|
76
|
+
(`{dir}/index.db` + `{dir}/{modality}/*.npz`, built with `Rollout.save()` +
|
|
77
|
+
`RolloutIndex.add()` — see `tests/test_index.py`). See `src/worldgap/cli.py`'s module
|
|
78
|
+
docstring for why this differs slightly from spec 5.3's single-shared-index diagram.
|
|
79
|
+
|
|
80
|
+
## Why not a webapp
|
|
81
|
+
|
|
82
|
+
This is infra meant to be dropped into someone else's pipeline, not a hosted service.
|
|
83
|
+
See spec Section 9 for the full API/CLI contract.
|
|
84
|
+
|
|
85
|
+
## License
|
|
86
|
+
|
|
87
|
+
MIT — see [`LICENSE`](LICENSE).
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# Roadmap
|
|
2
|
+
|
|
3
|
+
Phases per [`docs/TECHNICAL_SPEC.md`](docs/TECHNICAL_SPEC.md) Section 16. Checkboxes
|
|
4
|
+
reflect actual status, not aspiration — update this file every session, not just
|
|
5
|
+
at the end of a phase.
|
|
6
|
+
|
|
7
|
+
## Phase 0 — Data audit
|
|
8
|
+
- [x] Confirm HaGRID canonical-gesture-subset sample counts are sufficient —
|
|
9
|
+
resolved without needing Kaggle access: all four names in
|
|
10
|
+
`CANONICAL_GESTURES` are confirmed real HaGRID v1 class names, and the
|
|
11
|
+
dataset's own paper (Kapitanov et al. 2022, arXiv:2206.08219) states
|
|
12
|
+
each of the 18 classes contains 30,000+ images. Still open, and this
|
|
13
|
+
part *does* need real data: whether these four gestures are the best
|
|
14
|
+
semantic match to the ForceHand glove's actual DOFs.
|
|
15
|
+
- [ ] Confirm Ogawa et al. (2017) is accessible (open access, or via institutional
|
|
16
|
+
library) — flagged in spec Section 14 as the one real external dependency.
|
|
17
|
+
Checked so far: it's published in *Advanced Robotics* (Taylor & Francis,
|
|
18
|
+
subscription), and the only copy found online is a ResearchGate
|
|
19
|
+
"Request PDF" listing, not an open one — likely needs institutional
|
|
20
|
+
access or emailing the Kurita Lab directly.
|
|
21
|
+
|
|
22
|
+
## Phase 1 — Rollout schema, storage, loaders, perturbation
|
|
23
|
+
- [x] `Rollout` schema + save/load round-trip (`data/rollout.py`) — tested
|
|
24
|
+
- [x] Synthetic perturbation pipeline: tremor, reduced ROM, occlusion
|
|
25
|
+
(`data/loaders/synthetic_perturb.py`) — tested
|
|
26
|
+
- [x] HaGRID/EgoHands directory scanning + canonical gesture filtering — tested
|
|
27
|
+
(network-free portion only)
|
|
28
|
+
- [ ] HaGRID/EgoHands MediaPipe landmark extraction — **blocked**: needs real
|
|
29
|
+
downloaded frames + the `perception` extra, neither available in the
|
|
30
|
+
sandbox that produced this scaffolding. Currently a `NotImplementedError`
|
|
31
|
+
stub with the exact contract documented.
|
|
32
|
+
- [x] SQLite metadata index (spec 5.3) — `data/index.py`, tested
|
|
33
|
+
(documented decision: indexed per rollout-store directory, not one
|
|
34
|
+
shared repo-wide index — see `cli.py` module docstring)
|
|
35
|
+
|
|
36
|
+
## Phase 2 — World model core
|
|
37
|
+
- [x] `LandmarkEncoder` (Transformer, spec 6.1) — tested via forward pass
|
|
38
|
+
- [x] `ActuatorEncoder` (TCN, spec 6.2) — tested via forward pass
|
|
39
|
+
- [x] Shared `WorldModel` (JEPA-style core, EMA target encoder, spec 6.3) — tested
|
|
40
|
+
- [x] `CollapseSafeguard` — tested, both in isolation and wired into `fit()`
|
|
41
|
+
- [ ] Real convergence check on actual (non-synthetic-placeholder) data — blocked
|
|
42
|
+
on Phase 0/1 data access
|
|
43
|
+
|
|
44
|
+
## Phase 3 — Divergence module
|
|
45
|
+
- [x] Fréchet distance with Ledoit-Wolf shrinkage + complex-component handling
|
|
46
|
+
(spec 7.1) — tested, including the sample-size confidence flag (7.3)
|
|
47
|
+
- [x] MMD cross-check (spec 7.2) — tested
|
|
48
|
+
|
|
49
|
+
## Phase 4 — Validation harness
|
|
50
|
+
- [x] `ValidationHarness` with enforced pre-registration (anti-cherry-picking,
|
|
51
|
+
spec 8.3) — tested
|
|
52
|
+
- [x] Spearman + bootstrap CI (spec 8.2) — tested
|
|
53
|
+
- [ ] Actual V1 validation run against real MediaPipe-confidence ground truth —
|
|
54
|
+
blocked on Phase 0/1 data access
|
|
55
|
+
|
|
56
|
+
## Phase 5 — Packaging, CLI, demo notebook
|
|
57
|
+
- [x] `pyproject.toml`, src-layout, editable install — verified working
|
|
58
|
+
- [x] `GapAnalyzer` public API (spec 9.1) — tested across both modalities
|
|
59
|
+
(`tests/test_modality_swap.py` — the concrete reusability check)
|
|
60
|
+
- [x] CLI (`worldgap train/analyze/validate`) — wired end-to-end against local
|
|
61
|
+
rollout stores; verified via `tests/test_cli.py` (train->analyze
|
|
62
|
+
round trip, empty/missing-store errors, validate join + anti-cherry-pick
|
|
63
|
+
rejection) and against the real installed console-script entry point.
|
|
64
|
+
Still out of scope: producing rollout stores from raw HaGRID/EgoHands
|
|
65
|
+
frames in the first place (Phase 1's MediaPipe blocker)
|
|
66
|
+
- [x] Demo notebook (`notebooks/demo.ipynb`) — executes top-to-bottom via
|
|
67
|
+
`jupyter nbconvert --execute` with zero manual intervention (acceptance
|
|
68
|
+
criterion, spec Section 13); covers V1, the V1/V2 reusability claim, and
|
|
69
|
+
the validation harness's anti-cherry-picking rejection, entirely on
|
|
70
|
+
synthetic data
|
|
71
|
+
- [x] Report generation (spec 9.3) — `report.py`, HTML and Markdown output,
|
|
72
|
+
tested; includes the spec-210 Frechet/MMD rank-disagreement diagnostic
|
|
73
|
+
|
|
74
|
+
## Phase 6 — V2 actuation gap
|
|
75
|
+
- [x] Two-branch hysteresis-aware curve fit (spec 5.5, edge case 12.13) — tested
|
|
76
|
+
on synthetic hysteresis data, including a test that the naive
|
|
77
|
+
single-branch mistake is actually caught by the residual-structure check
|
|
78
|
+
- [ ] Digitize real Ogawa et al. curve via WebPlotDigitizer — blocked on Phase 0
|
|
79
|
+
- [ ] `test_modality_swap.py`-style end-to-end run on real PGM data — blocked
|
|
80
|
+
|
|
81
|
+
## Phase 7 — V3 (deferred)
|
|
82
|
+
- [ ] Not started. Contingent on Kurita Lab access (spec Section 15). What
|
|
83
|
+
changes and what doesn't is already documented there — no new design work
|
|
84
|
+
needed until access exists, only a new data loader.
|
|
85
|
+
|
|
86
|
+
---
|
|
87
|
+
|
|
88
|
+
## What's actually done vs. what's scaffolded
|
|
89
|
+
|
|
90
|
+
**Genuinely implemented and tested** (71 passing tests as of this writing):
|
|
91
|
+
Rollout schema, SQLite metadata index, synthetic perturbation, Fréchet + MMD
|
|
92
|
+
metrics, EMA, collapse safeguard, both encoders, the shared World Model Core,
|
|
93
|
+
`GapAnalyzer` end-to-end across both modalities (including checkpoint
|
|
94
|
+
save/load), the validation harness's anti-cherry-picking enforcement, the PGM
|
|
95
|
+
hysteresis fit (on synthetic data), HTML/Markdown report generation, the full
|
|
96
|
+
CLI (`train`/`analyze`/`validate`) against local rollout stores, and a demo
|
|
97
|
+
notebook that runs top-to-bottom on synthetic data with zero manual steps.
|
|
98
|
+
|
|
99
|
+
**Still blocked, not skipped for convenience**: the HaGRID/EgoHands MediaPipe
|
|
100
|
+
extraction step (needs real downloaded frames + network access this sandbox
|
|
101
|
+
doesn't have) and digitizing the real Ogawa et al. (2017) curve (needs a
|
|
102
|
+
likely-paywalled Taylor & Francis figure). Both have a clearly marked seam in
|
|
103
|
+
the code rather than a silent gap. Everything else in Phase 5 that doesn't
|
|
104
|
+
require those two inputs is done.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Default config for V1 (perception gap). Loaded via `worldgap train --config`.
|
|
2
|
+
# modality is set by the --modality CLI flag, not here (spec 9.2) -- this file
|
|
3
|
+
# starts from GapConfig's own defaults for everything else.
|
|
4
|
+
|
|
5
|
+
state_dim: 258 # 33*4 (pose) + 21*3 + 21*3 (hands), per spec 5.1.1
|
|
6
|
+
|
|
7
|
+
encoder:
|
|
8
|
+
d_model: 256
|
|
9
|
+
n_layers: 4
|
|
10
|
+
n_heads: 4
|
|
11
|
+
dim_feedforward: 1024
|
|
12
|
+
dropout: 0.1
|
|
13
|
+
|
|
14
|
+
world_model:
|
|
15
|
+
context_frames: 16 # ~0.5s at 30fps
|
|
16
|
+
predict_frames: 8 # ~0.25s at 30fps
|
|
17
|
+
ema_decay: 0.996
|
|
18
|
+
summary_dim: 64 # kept small deliberately -- spec 7.3
|
|
19
|
+
collapse_variance_threshold: 0.0001
|
|
20
|
+
collapse_patience_checks: 3
|
|
21
|
+
|
|
22
|
+
training:
|
|
23
|
+
lr: 0.0003
|
|
24
|
+
weight_decay: 0.01
|
|
25
|
+
batch_size: 64
|
|
26
|
+
max_epochs: 100
|
|
27
|
+
seed: 0
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Default config for V2 (actuation gap). Loaded via `worldgap train --config`.
|
|
2
|
+
# state_dim MUST be set explicitly for actuation (GapConfig raises otherwise,
|
|
3
|
+
# per spec 5.1.1) -- 2 here assumes the minimum-viable single-DOF case
|
|
4
|
+
# (wrist flexion): [commanded_pressure, resulting_joint_angle]. Bump this if
|
|
5
|
+
# modeling more DOFs.
|
|
6
|
+
|
|
7
|
+
state_dim: 2
|
|
8
|
+
|
|
9
|
+
encoder:
|
|
10
|
+
d_model: 64 # smaller than V1's -- spec 6.2, much lower input dimensionality
|
|
11
|
+
n_layers: 2
|
|
12
|
+
n_heads: 2
|
|
13
|
+
dim_feedforward: 128
|
|
14
|
+
dropout: 0.1
|
|
15
|
+
|
|
16
|
+
world_model:
|
|
17
|
+
context_frames: 16
|
|
18
|
+
predict_frames: 8
|
|
19
|
+
ema_decay: 0.996
|
|
20
|
+
summary_dim: 32 # smaller than V1's 64 -- fewer samples expected for V2, spec 7.3
|
|
21
|
+
collapse_variance_threshold: 0.0001
|
|
22
|
+
collapse_patience_checks: 3
|
|
23
|
+
|
|
24
|
+
training:
|
|
25
|
+
lr: 0.0003
|
|
26
|
+
weight_decay: 0.01
|
|
27
|
+
batch_size: 32
|
|
28
|
+
max_epochs: 100
|
|
29
|
+
seed: 0
|