wsms 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wsms-0.1.0/.gitignore +13 -0
- wsms-0.1.0/LICENSE +19 -0
- wsms-0.1.0/PKG-INFO +445 -0
- wsms-0.1.0/PYPI_README.md +341 -0
- wsms-0.1.0/WSM_v1/configs/wsg_v1_modalities_custom.json +120 -0
- wsms-0.1.0/WSM_v1/configs/wsg_v1_modalities_dense.json +3138 -0
- wsms-0.1.0/WSM_v1/configs/wsg_v1_modalities_sparse.json +121 -0
- wsms-0.1.0/WSM_v1/configs/wsm_v1_recommended_dense_channel_heads.json +62791 -0
- wsms-0.1.0/WSM_v1/scripts/infer_TunnelingNet_Flux2_4B_wsg.py +1390 -0
- wsms-0.1.0/WSM_v1/scripts/train_TunnelingNet_Flux2_4B_wsg.py +4685 -0
- wsms-0.1.0/WSM_v1/src/dataset/README.md +1 -0
- wsms-0.1.0/WSM_v1/src/dataset/__init__.py +1 -0
- wsms-0.1.0/WSM_v1/src/dataset/util.py +42 -0
- wsms-0.1.0/WSM_v1/src/dataset/wsg_dataset.py +2231 -0
- wsms-0.1.0/WSM_v1/src/pipelines/categorical_refiner.py +3316 -0
- wsms-0.1.0/WSM_v1/src/pipelines/wsg_sparse_codecs/__init__.py +10 -0
- wsms-0.1.0/WSM_v1/src/pipelines/wsg_sparse_codecs/dwpose_heatmap.py +566 -0
- wsms-0.1.0/WSM_v1/src/pipelines/wsg_sparse_codecs/registry.py +115 -0
- wsms-0.1.0/WSM_v1/src/visualization/__init__.py +1 -0
- wsms-0.1.0/WSM_v1/src/visualization/single_wsg_visualization.py +768 -0
- wsms-0.1.0/WSM_v1/third_party/ScoreHMR/wsm_adapters/build_wsv_audio_dataset.py +703 -0
- wsms-0.1.0/WSM_v1/third_party/ScoreHMR/wsm_adapters/wsv_smpl_utils.py +372 -0
- wsms-0.1.0/WSM_v1/webui/__init__.py +1 -0
- wsms-0.1.0/WSM_v1/webui/input_validation.py +768 -0
- wsms-0.1.0/WSM_v1/wsg_reader/categorical_visualization.py +565 -0
- wsms-0.1.0/WSM_v1/wsg_reader/configs/channel_visualization_palettes.json +542 -0
- wsms-0.1.0/WSM_v1/wsg_reader/normalized_wsg_loader.py +230 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline01_read_dwpose_from_wsg.py +423 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline02_read_smplerx_from_wsg.py +2313 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline03_read_mediapipe_from_wsg.py +348 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline04c_read_mediapipe_segmentation_from_wsg.py +231 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline05_read_primedepth_from_wsg.py +215 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline06_read_metricdepth_from_wsg.py +211 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline07_read_normals_from_wsg.py +387 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline08_read_curvature_from_wsg.py +225 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline09_read_edge_from_wsg.py +188 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline10_read_opticalflow_from_wsg.py +294 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline14_read_xpose_pose_visualization_from_wsg.py +133 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline15_read_object_from_wsg.py +1424 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline16_read_psy_from_wsg.py +3095 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline17_read_PBR_from_wsg.py +413 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline18_read_audio_from_wsg.py +1100 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline19_read_caption_audio_from_wsg.py +749 -0
- wsms-0.1.0/WSM_v1/wsg_reader/pipeline20_read_MMFi_from_wsg.py +443 -0
- wsms-0.1.0/WSM_v1/wsg_reader/tools/Objaverse/category_id_mapping.json +115 -0
- wsms-0.1.0/WSM_v1/wsg_reader/tools/ShapeNet/category_id_mapping.json +372 -0
- wsms-0.1.0/WSM_v1/wsg_reader/tools/ShapeNet/material_id_mapping.json +46 -0
- wsms-0.1.0/WSM_v1/wsg_reader/utils/binvox_relative.py +163 -0
- wsms-0.1.0/WSM_v1/wsg_reader/utils/binvox_utils.py +163 -0
- wsms-0.1.0/WSM_v1/wsg_reader/utils/blender_script.py +1048 -0
- wsms-0.1.0/WSM_v1/wsg_reader/utils/blender_script_shapenet.py +1136 -0
- wsms-0.1.0/WSM_v1/wsg_reader/utils/draw_dw_lib.py +358 -0
- wsms-0.1.0/WSM_v1/wsg_reader/utils/optimized_processing.py +327 -0
- wsms-0.1.0/WSM_v1/wsg_reader/utils/preprocess_video.py +259 -0
- wsms-0.1.0/WSM_v1/wsg_reader/utils/util.py +128 -0
- wsms-0.1.0/pyproject.toml +214 -0
- wsms-0.1.0/src/examples/WSM_Quickstart.ipynb +4630 -0
- wsms-0.1.0/src/wsms/__init__.py +30 -0
- wsms-0.1.0/src/wsms/catalog.py +291 -0
- wsms-0.1.0/src/wsms/cli.py +334 -0
- wsms-0.1.0/src/wsms/hub.py +142 -0
- wsms-0.1.0/src/wsms/outputs.py +22 -0
- wsms-0.1.0/src/wsms/pipeline.py +309 -0
- wsms-0.1.0/src/wsms/runtime.py +95 -0
- wsms-0.1.0/src/wsms/version.py +3 -0
wsms-0.1.0/.gitignore
ADDED
wsms-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
WSM Non-commercial Research License
|
|
2
|
+
|
|
3
|
+
© 2026–2027 Rutgers, The State University of New Jersey.
|
|
4
|
+
|
|
5
|
+
The accompanying source code, executable software and binaries, documentation, and Rutgers-owned model artifacts (including trained or fine-tuned model weights, checkpoints, adapters, model.pth files, and other model-weight files) are made available under the terms of this Rutgers’ Non-commercial Research License (this “RU-NCRL”). By downloading, installing, or using any of these materials, you are consenting to be bound by this RU-NCRL. If you do not agree to the terms and conditions of this license, do not download, install, or use any of these materials.
|
|
6
|
+
|
|
7
|
+
Redistribution and use for noncommercial purposes in source, binary, and model-artifact forms, with or without modification, are permitted provided that the following conditions are met:
|
|
8
|
+
|
|
9
|
+
1. The software is used solely for academic, research, experimental, personal noncommercial purposes. It may not be used indirectly for commercial use, such as on a website that accepts advertising money for content. For commercial use rights, contact Rutgers, The State University of New Jersey, at compliance@research.rutgers.edu and inquire about the Rutgers Docket #2027-020.
|
|
10
|
+
|
|
11
|
+
2. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
|
12
|
+
|
|
13
|
+
3. Redistributions in binary or model-artifact form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
4. Neither the name of Rutgers, The State University of New Jersey nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
|
16
|
+
|
|
17
|
+
This Agreement shall be governed by, construed, interpreted and enforced under the laws of the State of New Jersey, except for its conflict of laws principles. The venue shall be the state and federal courts located in the State of New Jersey.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE RUTGERS, THE STATE UNIVERSITY OF NEW JERSEY AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RUTGERS, THE STATE UNIVERSITY OF NEW JERSEY OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
wsms-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,445 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: wsms
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: World Snapshot Models pipelines for unified visual generation, estimation, control, and reconstruction
|
|
5
|
+
Project-URL: Homepage, https://world-snapshot.github.io/WSMs/
|
|
6
|
+
Project-URL: Documentation, https://world-snapshot.github.io/doc/index.html
|
|
7
|
+
Project-URL: Repository, https://github.com/World-Snapshot/WSMs
|
|
8
|
+
Project-URL: Paper, https://arxiv.org/abs/2611.24917
|
|
9
|
+
Project-URL: Models, https://huggingface.co/World-Snapshot/WSM_v1
|
|
10
|
+
Project-URL: Datasets, https://huggingface.co/World-Snapshot/datasets
|
|
11
|
+
Project-URL: Issues, https://github.com/World-Snapshot/WSMs/issues
|
|
12
|
+
Author: World Snapshot
|
|
13
|
+
Author-email: Sen Fang <sen.fang@rutgers.edu>
|
|
14
|
+
License: WSM Non-commercial Research License
|
|
15
|
+
|
|
16
|
+
© 2026–2027 Rutgers, The State University of New Jersey.
|
|
17
|
+
|
|
18
|
+
The accompanying source code, executable software and binaries, documentation, and Rutgers-owned model artifacts (including trained or fine-tuned model weights, checkpoints, adapters, model.pth files, and other model-weight files) are made available under the terms of this Rutgers’ Non-commercial Research License (this “RU-NCRL”). By downloading, installing, or using any of these materials, you are consenting to be bound by this RU-NCRL. If you do not agree to the terms and conditions of this license, do not download, install, or use any of these materials.
|
|
19
|
+
|
|
20
|
+
Redistribution and use for noncommercial purposes in source, binary, and model-artifact forms, with or without modification, are permitted provided that the following conditions are met:
|
|
21
|
+
|
|
22
|
+
1. The software is used solely for academic, research, experimental, personal noncommercial purposes. It may not be used indirectly for commercial use, such as on a website that accepts advertising money for content. For commercial use rights, contact Rutgers, The State University of New Jersey, at compliance@research.rutgers.edu and inquire about the Rutgers Docket #2027-020.
|
|
23
|
+
|
|
24
|
+
2. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
|
25
|
+
|
|
26
|
+
3. Redistributions in binary or model-artifact form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
|
27
|
+
|
|
28
|
+
4. Neither the name of Rutgers, The State University of New Jersey nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
|
29
|
+
|
|
30
|
+
This Agreement shall be governed by, construed, interpreted and enforced under the laws of the State of New Jersey, except for its conflict of laws principles. The venue shall be the state and federal courts located in the State of New Jersey.
|
|
31
|
+
|
|
32
|
+
THIS SOFTWARE IS PROVIDED BY THE RUTGERS, THE STATE UNIVERSITY OF NEW JERSEY AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RUTGERS, THE STATE UNIVERSITY OF NEW JERSEY OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
33
|
+
License-File: LICENSE
|
|
34
|
+
Keywords: computer vision,diffusers,generative model,multimodal,tunnelingnet,world snapshot model
|
|
35
|
+
Classifier: Development Status :: 3 - Alpha
|
|
36
|
+
Classifier: Intended Audience :: Science/Research
|
|
37
|
+
Classifier: Programming Language :: Python :: 3
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
39
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
40
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
41
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
42
|
+
Requires-Python: >=3.10
|
|
43
|
+
Requires-Dist: accelerate>=1.2.0
|
|
44
|
+
Requires-Dist: av>=13.0.0
|
|
45
|
+
Requires-Dist: diffusers>=0.40.0
|
|
46
|
+
Requires-Dist: einops>=0.8.0
|
|
47
|
+
Requires-Dist: huggingface-hub>=0.36.0
|
|
48
|
+
Requires-Dist: imageio[ffmpeg]>=2.36.0
|
|
49
|
+
Requires-Dist: lightning[pytorch-extra]<2.6,>=2.1
|
|
50
|
+
Requires-Dist: matplotlib>=3.9.3
|
|
51
|
+
Requires-Dist: numba>=0.60.0
|
|
52
|
+
Requires-Dist: numpy<3,>=1.26.4
|
|
53
|
+
Requires-Dist: opencv-python-headless>=4.10.0
|
|
54
|
+
Requires-Dist: optimum-quanto>=0.2.6
|
|
55
|
+
Requires-Dist: packaging>=24.0
|
|
56
|
+
Requires-Dist: pandas>=2.2.3
|
|
57
|
+
Requires-Dist: peft>=0.18.0
|
|
58
|
+
Requires-Dist: pillow>=10.0.0
|
|
59
|
+
Requires-Dist: plyfile>=1.1
|
|
60
|
+
Requires-Dist: pyrender>=0.1.45
|
|
61
|
+
Requires-Dist: pyyaml>=6.0
|
|
62
|
+
Requires-Dist: safetensors>=0.5.0
|
|
63
|
+
Requires-Dist: scikit-learn>=1.5.0
|
|
64
|
+
Requires-Dist: scipy>=1.14.0
|
|
65
|
+
Requires-Dist: sentencepiece>=0.2.0
|
|
66
|
+
Requires-Dist: speechrecognition>=3.10.0
|
|
67
|
+
Requires-Dist: sympy>=1.13.0
|
|
68
|
+
Requires-Dist: timm>=1.0.12
|
|
69
|
+
Requires-Dist: torch>=2.5.0
|
|
70
|
+
Requires-Dist: torchvision>=0.20.0
|
|
71
|
+
Requires-Dist: tqdm>=4.66.0
|
|
72
|
+
Requires-Dist: transformers>=5.0.0
|
|
73
|
+
Requires-Dist: trimesh>=4.5.0
|
|
74
|
+
Provides-Extra: audio
|
|
75
|
+
Requires-Dist: openai-whisper>=20240930; extra == 'audio'
|
|
76
|
+
Provides-Extra: dev
|
|
77
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
78
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
79
|
+
Requires-Dist: twine>=6; extra == 'dev'
|
|
80
|
+
Provides-Extra: full
|
|
81
|
+
Requires-Dist: bitsandbytes>=0.45; extra == 'full'
|
|
82
|
+
Requires-Dist: fastapi>=0.115; extra == 'full'
|
|
83
|
+
Requires-Dist: gradio>=4; extra == 'full'
|
|
84
|
+
Requires-Dist: mayavi>=4.8; extra == 'full'
|
|
85
|
+
Requires-Dist: open3d>=0.18; extra == 'full'
|
|
86
|
+
Requires-Dist: openai-whisper>=20240930; extra == 'full'
|
|
87
|
+
Requires-Dist: plotly>=5; extra == 'full'
|
|
88
|
+
Requires-Dist: uvicorn>=0.32; extra == 'full'
|
|
89
|
+
Requires-Dist: viser>=0.2; extra == 'full'
|
|
90
|
+
Requires-Dist: wandb>=0.19; extra == 'full'
|
|
91
|
+
Requires-Dist: xformers>=0.0.29; extra == 'full'
|
|
92
|
+
Provides-Extra: training
|
|
93
|
+
Requires-Dist: bitsandbytes>=0.45; extra == 'training'
|
|
94
|
+
Requires-Dist: wandb>=0.19; extra == 'training'
|
|
95
|
+
Requires-Dist: xformers>=0.0.29; extra == 'training'
|
|
96
|
+
Provides-Extra: webui
|
|
97
|
+
Requires-Dist: fastapi>=0.115; extra == 'webui'
|
|
98
|
+
Requires-Dist: gradio>=4; extra == 'webui'
|
|
99
|
+
Requires-Dist: open3d>=0.18; extra == 'webui'
|
|
100
|
+
Requires-Dist: plotly>=5; extra == 'webui'
|
|
101
|
+
Requires-Dist: uvicorn>=0.32; extra == 'webui'
|
|
102
|
+
Requires-Dist: viser>=0.2; extra == 'webui'
|
|
103
|
+
Description-Content-Type: text/markdown
|
|
104
|
+
|
|
105
|
+
# WSM: World Snapshot Models
|
|
106
|
+
|
|
107
|
+
**World Snapshot Models (WSMs) are an ideal world model designed to unify,
|
|
108
|
+
expand, and ultimately transcend Computer Vision.** The first implementation is
|
|
109
|
+
WSM v1, codename Augustus.
|
|
110
|
+
|
|
111
|
+
To train the world itself, we define world frames and use
|
|
112
|
+
[World Snapshot Graphics (`.wsg`)](https://github.com/World-Snapshot/wsg) to
|
|
113
|
+
encode RGB, geometry, motion, pose, physics, materials, semantics, audio, and
|
|
114
|
+
other information into a unified 256-channel image. This realizes the
|
|
115
|
+
[Unified Law for Visual Tasks](https://github.com/World-Snapshot/Unified-Law-for-Visual-Tasks):
|
|
116
|
+
**if all information can be encoded into the image itself, then two-dimensional
|
|
117
|
+
image generation becomes lossless world generation.**
|
|
118
|
+
|
|
119
|
+
WSM v1 is the first large-scale implementation of this law. **A single
|
|
120
|
+
generative model validates 1,148 visual tasks and provides a trained baseline
|
|
121
|
+
for every task.** Capabilities previously scattered across different models,
|
|
122
|
+
datasets, and research communities coexist in the same system, while 99.99% of
|
|
123
|
+
earlier visual models were built for one task. Papers with Code recorded 1,477
|
|
124
|
+
visual or vision-related tasks representing the existing field; all 1,477 are
|
|
125
|
+
compatible with WSM interfaces. Among the 1,148 evaluated tasks, an estimated
|
|
126
|
+
655–1,056 are entirely new, allowing WSM to **expand the recorded Computer
|
|
127
|
+
Vision task space by 44.3–71.5%.**
|
|
128
|
+
|
|
129
|
+
The four WSM interfaces are the minimum common training form of the complete
|
|
130
|
+
visual task space. At this higher level, classification, localization,
|
|
131
|
+
detection, segmentation, estimation, reconstruction, restoration, completion,
|
|
132
|
+
registration, correspondence, tracking, retrieval, indexing, prediction,
|
|
133
|
+
generation, editing, rendering, simulation, planning, and control all belong
|
|
134
|
+
to these four interface categories. Thousands of concrete tasks can therefore
|
|
135
|
+
be trained through the same four unified interfaces.
|
|
136
|
+
|
|
137
|
+
This scale is enabled by **TunnelingNet**. It preserves a shared FLUX.2
|
|
138
|
+
generative backbone while learning an independently trainable compact tunnel
|
|
139
|
+
for each directed task. Every tunnel can be trained, loaded, improved,
|
|
140
|
+
separated, or newly added without retraining the task bank from scratch.
|
|
141
|
+
Compared with parameter-matched multimodal and unified-model baselines,
|
|
142
|
+
TunnelingNet reveals recognizable structure 77–100 times earlier and requires
|
|
143
|
+
19.0–38.5 times fewer optimizer updates. Each task adds only a
|
|
144
|
+
12.51M-parameter tunnel; with the complete task bank loaded, WSM v1 forms an
|
|
145
|
+
18B-class model rather than multiplying the backbone thousands of times.
|
|
146
|
+
|
|
147
|
+
The `wsms` package makes this complete task space directly callable through
|
|
148
|
+
`WSMPipeline`, command-line discovery, and qualified Dense, Sparse, and
|
|
149
|
+
Custom Channel Heads.
|
|
150
|
+
|
|
151
|
+
## Installation
|
|
152
|
+
|
|
153
|
+
```bash
|
|
154
|
+
pip install wsms
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
The [one-click WSM notebook](https://github.com/World-Snapshot/WSMs/blob/main/src/examples/WSM_Quickstart.ipynb)
|
|
158
|
+
turns the discovery and inference examples below into an executable GPU
|
|
159
|
+
workflow.
|
|
160
|
+
|
|
161
|
+
WSM v1 inference requires a CUDA-capable GPU and currently runs in
|
|
162
|
+
`torch.bfloat16`. If pip selects a PyTorch build that does not match the local
|
|
163
|
+
CUDA driver, install the appropriate PyTorch wheel first and then install
|
|
164
|
+
`wsms`.
|
|
165
|
+
|
|
166
|
+
Optional interfaces can be installed without changing the API:
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
pip install "wsms[audio]" # audio-specific decoding support
|
|
170
|
+
pip install "wsms[webui]" # ControlWorld/WebUI dependencies
|
|
171
|
+
pip install "wsms[training]" # training and continuation utilities
|
|
172
|
+
pip install "wsms[full]" # all optional components
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
The pip distribution contains the WSM pipeline, task catalog, modality
|
|
176
|
+
contracts, and the source required to run released Channel Heads. Model weights
|
|
177
|
+
are downloaded separately from Hugging Face and are never embedded in the
|
|
178
|
+
Python package.
|
|
179
|
+
|
|
180
|
+
### Install directly from GitHub
|
|
181
|
+
|
|
182
|
+
The public Git repository can also be installed directly. Dependencies declared
|
|
183
|
+
in `pyproject.toml` are installed automatically:
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
pip install "wsms @ git+https://github.com/World-Snapshot/WSMs.git@v0.1.0"
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
Replace the release tag with a specific commit when exact reproducibility is
|
|
190
|
+
required.
|
|
191
|
+
|
|
192
|
+
## Model access
|
|
193
|
+
|
|
194
|
+
The default model repository is:
|
|
195
|
+
|
|
196
|
+
```text
|
|
197
|
+
World-Snapshot/WSM_v1
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
The model repository is public, so downloading released checkpoints does not
|
|
201
|
+
require authentication. A Hugging Face login remains optional for users who
|
|
202
|
+
want authenticated Hub access:
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
hf auth login
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
When present, the pipeline uses the cached Hugging Face identity automatically.
|
|
209
|
+
It also supports the standard `HF_TOKEN` environment variable and an explicit
|
|
210
|
+
`token=...` argument. Avoid storing tokens in source files or shell history.
|
|
211
|
+
|
|
212
|
+
## Three levels of discovery
|
|
213
|
+
|
|
214
|
+
WSM separates the public task relationship from its concrete implementation:
|
|
215
|
+
|
|
216
|
+
1. A **tunnel** is an ordered input-to-output mapping, such as
|
|
217
|
+
`0-2 -> 36-38`.
|
|
218
|
+
2. A **Head** is one implementation of that mapping.
|
|
219
|
+
3. A **family** qualifies the Head namespace: `dense`, `sparse`, or
|
|
220
|
+
`custom`.
|
|
221
|
+
|
|
222
|
+
Head indices repeat across families. Consequently, a bare value such as
|
|
223
|
+
`000003` is intentionally rejected. Use `dense:000003`,
|
|
224
|
+
`sparse:000003`, or `custom:000003` when pinning an implementation.
|
|
225
|
+
|
|
226
|
+
### Command-line discovery
|
|
227
|
+
|
|
228
|
+
```bash
|
|
229
|
+
wsms list
|
|
230
|
+
wsms tunnel list
|
|
231
|
+
wsms head list
|
|
232
|
+
wsms info
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
`wsms list` shows the package's top-level capabilities. `wsms tunnel list`
|
|
236
|
+
shows all user-facing mappings, while `wsms head list` shows the qualified
|
|
237
|
+
implementation behind each mapping.
|
|
238
|
+
|
|
239
|
+
Examples:
|
|
240
|
+
|
|
241
|
+
```bash
|
|
242
|
+
# RGB -> surface normals
|
|
243
|
+
wsms tunnel list --input 0-2 --output 36-38
|
|
244
|
+
|
|
245
|
+
# Ordered multi-input route
|
|
246
|
+
wsms tunnel list --input 39 --input 0-2 --output 90-92
|
|
247
|
+
|
|
248
|
+
# Only Sparse implementations
|
|
249
|
+
wsms head list --source sparse
|
|
250
|
+
|
|
251
|
+
# Machine-readable discovery
|
|
252
|
+
wsms tunnel list --output Universal_Pose --json
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
Filters preserve input order because `A + B -> C` and `B + A -> C` can
|
|
256
|
+
represent different routing contracts.
|
|
257
|
+
|
|
258
|
+
### Python discovery
|
|
259
|
+
|
|
260
|
+
```python
|
|
261
|
+
from wsms import WSMPipeline
|
|
262
|
+
|
|
263
|
+
tasks = WSMPipeline.list_tasks(
|
|
264
|
+
inputs=["0-2"],
|
|
265
|
+
output="36-38",
|
|
266
|
+
)
|
|
267
|
+
print(tasks[0]["task_id"])
|
|
268
|
+
print(tasks[0]["head"])
|
|
269
|
+
print(tasks[0]["condition_description"])
|
|
270
|
+
print(tasks[0]["target_description"])
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
The catalog combines all released and recommended Dense, Sparse, and Custom
|
|
274
|
+
Heads. It can be queried without downloading model weights.
|
|
275
|
+
|
|
276
|
+
## Python inference
|
|
277
|
+
|
|
278
|
+
Input-output selection is the recommended public API:
|
|
279
|
+
|
|
280
|
+
```python
|
|
281
|
+
import torch
|
|
282
|
+
from wsms import WSMPipeline
|
|
283
|
+
|
|
284
|
+
pipe = WSMPipeline.from_pretrained(
|
|
285
|
+
"World-Snapshot/WSM_v1",
|
|
286
|
+
inputs=["0-2"],
|
|
287
|
+
output="36-38",
|
|
288
|
+
torch_dtype=torch.bfloat16,
|
|
289
|
+
device="cuda:0",
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
result = pipe(
|
|
293
|
+
"input.png",
|
|
294
|
+
num_inference_steps=10,
|
|
295
|
+
seed=20260812,
|
|
296
|
+
size=256,
|
|
297
|
+
)
|
|
298
|
+
result.images[0].save("normal.png")
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
`from_pretrained` downloads only the selected task run rather than every WSM
|
|
302
|
+
checkpoint. The FLUX.2 Klein 4B backbone is shared across repeated calls to the
|
|
303
|
+
same pipeline object.
|
|
304
|
+
|
|
305
|
+
### Multi-input tunnels
|
|
306
|
+
|
|
307
|
+
Pass a dictionary keyed by modality ID:
|
|
308
|
+
|
|
309
|
+
```python
|
|
310
|
+
pipe = WSMPipeline.from_pretrained(
|
|
311
|
+
"World-Snapshot/WSM_v1",
|
|
312
|
+
inputs=["39", "0-2"],
|
|
313
|
+
output="90-92",
|
|
314
|
+
device="cuda:0",
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
result = pipe(
|
|
318
|
+
{
|
|
319
|
+
"39": "curvature.npz",
|
|
320
|
+
"0-2": "reference.png",
|
|
321
|
+
}
|
|
322
|
+
)
|
|
323
|
+
```
|
|
324
|
+
|
|
325
|
+
An ordered list or tuple is also accepted, but a dictionary makes the routing
|
|
326
|
+
contract more explicit.
|
|
327
|
+
|
|
328
|
+
### Pinning a concrete Head
|
|
329
|
+
|
|
330
|
+
Use the input-output interface for ordinary use. A qualified Head is available
|
|
331
|
+
for exact reproduction:
|
|
332
|
+
|
|
333
|
+
```python
|
|
334
|
+
pipe = WSMPipeline.from_pretrained(
|
|
335
|
+
"World-Snapshot/WSM_v1",
|
|
336
|
+
head="custom:000000",
|
|
337
|
+
device="cuda:0",
|
|
338
|
+
)
|
|
339
|
+
```
|
|
340
|
+
|
|
341
|
+
### Local and offline loading
|
|
342
|
+
|
|
343
|
+
A local WSM release root or a direct checkpoint can replace the Hub repository:
|
|
344
|
+
|
|
345
|
+
```python
|
|
346
|
+
pipe = WSMPipeline.from_pretrained(
|
|
347
|
+
"/models/WSM_v1",
|
|
348
|
+
inputs=["0-2"],
|
|
349
|
+
output="36-38",
|
|
350
|
+
base_model="/models/FLUX.2-klein-4B",
|
|
351
|
+
device="cuda:0",
|
|
352
|
+
local_files_only=True,
|
|
353
|
+
)
|
|
354
|
+
```
|
|
355
|
+
|
|
356
|
+
Hugging Face's normal cache behavior applies. Set `cache_dir` to control
|
|
357
|
+
where downloaded artifacts are stored.
|
|
358
|
+
|
|
359
|
+
## Command-line inference
|
|
360
|
+
|
|
361
|
+
The command-line interface uses the same catalog and `WSMPipeline` runtime:
|
|
362
|
+
|
|
363
|
+
```bash
|
|
364
|
+
wsms infer \
|
|
365
|
+
--input 0-2=input.png \
|
|
366
|
+
--target 36-38 \
|
|
367
|
+
--device cuda:0 \
|
|
368
|
+
--steps 10 \
|
|
369
|
+
--seed 20260812 \
|
|
370
|
+
--output-dir outputs/rgb_to_normal
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
Multi-input example:
|
|
374
|
+
|
|
375
|
+
```bash
|
|
376
|
+
wsms infer \
|
|
377
|
+
--input 39=curvature.npz \
|
|
378
|
+
--input 0-2=reference.png \
|
|
379
|
+
--target 90-92 \
|
|
380
|
+
--device cuda:0 \
|
|
381
|
+
--output-dir outputs/curvature_reference_to_target
|
|
382
|
+
```
|
|
383
|
+
|
|
384
|
+
To pin an implementation, replace `--target` with a qualified selector such
|
|
385
|
+
as `--head dense:000424`. Run `wsms infer --help` for all runtime options.
|
|
386
|
+
|
|
387
|
+
## Accepted inputs
|
|
388
|
+
|
|
389
|
+
The Python pipeline accepts:
|
|
390
|
+
|
|
391
|
+
- paths to PNG/JPEG and WSG/NPZ inputs;
|
|
392
|
+
- `PIL.Image.Image` objects;
|
|
393
|
+
- NumPy arrays;
|
|
394
|
+
- PyTorch tensors.
|
|
395
|
+
|
|
396
|
+
RGB arrays are materialized as images. Non-RGB arrays are materialized as
|
|
397
|
+
modality-scoped NPZ files and then pass through the same normalization and
|
|
398
|
+
encoding contract used by the canonical WSM inference script.
|
|
399
|
+
|
|
400
|
+
## Outputs
|
|
401
|
+
|
|
402
|
+
Calling the pipeline returns `WSMPipelineOutput`:
|
|
403
|
+
|
|
404
|
+
```python
|
|
405
|
+
result.images # list[PIL.Image.Image], primary visual outputs
|
|
406
|
+
result.wsg # generated WSG array when available
|
|
407
|
+
result.task_id # resolved family and input-output mapping
|
|
408
|
+
result.artifacts # files retained when output_dir is provided
|
|
409
|
+
result.metadata # task, seed, steps, size, and runtime manifest
|
|
410
|
+
```
|
|
411
|
+
|
|
412
|
+
Without `output_dir`, temporary intermediate files are removed after the
|
|
413
|
+
in-memory result is loaded. With `output_dir`, the generated image, WSG,
|
|
414
|
+
manifest, and related runtime artifacts are retained.
|
|
415
|
+
|
|
416
|
+
## Special modality assets
|
|
417
|
+
|
|
418
|
+
Some geometry and human-body visualizations require separately licensed assets.
|
|
419
|
+
Set these only when the selected tunnel needs them:
|
|
420
|
+
|
|
421
|
+
- `WSM_SMPLX_MODEL_ROOT`: directory containing SMPL-X model assets;
|
|
422
|
+
- `WSM_HUMAN_MODELS_PYTHONPATH`: optional Python helper root for SMPL-X;
|
|
423
|
+
- `WSM_SHAPENET_TEXTURE_ROOT`: ShapeNet texture directory.
|
|
424
|
+
|
|
425
|
+
These third-party assets are not distributed by `wsms`.
|
|
426
|
+
|
|
427
|
+
## Naming compatibility
|
|
428
|
+
|
|
429
|
+
`WSMPipeline` follows Python and Diffusers class naming conventions.
|
|
430
|
+
`WSM_Pipeline` remains available as a compatibility alias:
|
|
431
|
+
|
|
432
|
+
```python
|
|
433
|
+
from wsms import WSMPipeline, WSM_Pipeline
|
|
434
|
+
|
|
435
|
+
assert WSMPipeline is WSM_Pipeline
|
|
436
|
+
```
|
|
437
|
+
|
|
438
|
+
## Links
|
|
439
|
+
|
|
440
|
+
- [Project](https://world-snapshot.github.io/WSMs/)
|
|
441
|
+
- [Source](https://github.com/World-Snapshot/WSMs)
|
|
442
|
+
- [Model repository](https://huggingface.co/World-Snapshot/WSM_v1)
|
|
443
|
+
- [Datasets](https://huggingface.co/World-Snapshot/datasets)
|
|
444
|
+
- [Documentation](https://world-snapshot.github.io/doc/index.html)
|
|
445
|
+
- [Paper](https://arxiv.org/abs/2611.24917)
|