vid2anim 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vid2anim/__init__.py +40 -0
- vid2anim/cli.py +242 -0
- vid2anim/core/__init__.py +1 -0
- vid2anim/core/auto_rig.py +134 -0
- vid2anim/core/bvh_writer.py +62 -0
- vid2anim/core/calibration.py +88 -0
- vid2anim/core/canonical_skeleton.py +400 -0
- vid2anim/core/glb_writer.py +270 -0
- vid2anim/core/interpolation.py +107 -0
- vid2anim/core/kinematics.py +197 -0
- vid2anim/core/motion_extractor.py +1354 -0
- vid2anim/core/retargeter.py +164 -0
- vid2anim/core/rig_detector.py +857 -0
- vid2anim/core/scale_normalizer.py +50 -0
- vid2anim/core/smoothing.py +195 -0
- vid2anim/core/universal_transfer.py +1203 -0
- vid2anim/pipeline.py +169 -0
- vid2anim/server.py +2662 -0
- vid2anim/utils/__init__.py +1 -0
- vid2anim/utils/cli_features.py +397 -0
- vid2anim/utils/diagnostics.py +88 -0
- vid2anim/utils/fixtures.py +334 -0
- vid2anim/utils/math_3d.py +140 -0
- vid2anim/utils/progress.py +66 -0
- vid2anim/utils/stabilize_kp2d.py +97 -0
- vid2anim-0.1.1.dist-info/METADATA +414 -0
- vid2anim-0.1.1.dist-info/RECORD +31 -0
- vid2anim-0.1.1.dist-info/WHEEL +5 -0
- vid2anim-0.1.1.dist-info/entry_points.txt +2 -0
- vid2anim-0.1.1.dist-info/licenses/LICENSE +21 -0
- vid2anim-0.1.1.dist-info/top_level.txt +1 -0
vid2anim/__init__.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""vid2anim: Animate any 3D model from photo or video.
|
|
2
|
+
|
|
3
|
+
Supports CPU (default/lightweight) and GPU acceleration.
|
|
4
|
+
Universal retargeting for Mixamo, VRM, Ready Player Me, Blender Rigify,
|
|
5
|
+
Unreal Engine, Daz3D/Character Creator, and experimental auto-rigging for static meshes.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
__version__ = "0.1.1"
|
|
11
|
+
|
|
12
|
+
from .core.canonical_skeleton import CANONICAL_JOINTS_ORDER, CanonicalMotion
|
|
13
|
+
from .core.motion_extractor import (
|
|
14
|
+
AsyncVideoReader,
|
|
15
|
+
MotionExtractor,
|
|
16
|
+
MotionSequence,
|
|
17
|
+
configure_isolated_environment,
|
|
18
|
+
)
|
|
19
|
+
from .core.retargeter import RetargetedAnimation, UniversalRetargeter
|
|
20
|
+
from .core.rig_detector import RigAnalysis, RigType, inspect_model
|
|
21
|
+
from .pipeline import PipelineConfig, PipelineResult, Vid2AnimPipeline
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"__version__",
|
|
25
|
+
"AsyncVideoReader",
|
|
26
|
+
"CANONICAL_JOINTS_ORDER",
|
|
27
|
+
"CanonicalMotion",
|
|
28
|
+
"MotionExtractor",
|
|
29
|
+
"MotionSequence",
|
|
30
|
+
"PipelineConfig",
|
|
31
|
+
"PipelineResult",
|
|
32
|
+
"RetargetedAnimation",
|
|
33
|
+
"RigAnalysis",
|
|
34
|
+
"RigType",
|
|
35
|
+
"UniversalRetargeter",
|
|
36
|
+
"Vid2AnimPipeline",
|
|
37
|
+
"configure_isolated_environment",
|
|
38
|
+
"inspect_model",
|
|
39
|
+
]
|
|
40
|
+
|
vid2anim/cli.py
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
"""Command Line Interface for vid2anim.
|
|
2
|
+
|
|
3
|
+
Commands:
|
|
4
|
+
vid2anim doctor Check hardware, CPU, CUDA GPU, and dependencies.
|
|
5
|
+
vid2anim inspect Inspect a 3D model and detect its skeleton/rig profile.
|
|
6
|
+
vid2anim extract Extract 3D motion from a video or photo to cached .npz.
|
|
7
|
+
vid2anim retarget Retarget cached motion onto a 3D model.
|
|
8
|
+
vid2anim serve Launch local Web UI & API server with 3D viewport.
|
|
9
|
+
vid2anim run End-to-end: input media + 3D model -> animated .glb!
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import argparse
|
|
15
|
+
import sys
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from . import __version__
|
|
19
|
+
from .core.glb_writer import GLBAnimationExporter
|
|
20
|
+
from .core.motion_extractor import MotionExtractor, MotionSequence
|
|
21
|
+
from .core.retargeter import UniversalRetargeter
|
|
22
|
+
from .core.rig_detector import RigType, inspect_model
|
|
23
|
+
from .utils.cli_features import (
|
|
24
|
+
handle_addon_command,
|
|
25
|
+
handle_batch_command,
|
|
26
|
+
handle_benchmark_command,
|
|
27
|
+
handle_info_command,
|
|
28
|
+
handle_samples_command,
|
|
29
|
+
)
|
|
30
|
+
from .utils.diagnostics import print_doctor_report
|
|
31
|
+
from .utils.progress import ProgressReporter
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def main() -> None:
|
|
35
|
+
parser = argparse.ArgumentParser(
|
|
36
|
+
prog="vid2anim",
|
|
37
|
+
description="Animate any 3D model from photo or video with AI motion capture and universal retargeting.",
|
|
38
|
+
)
|
|
39
|
+
parser.add_argument(
|
|
40
|
+
"-v",
|
|
41
|
+
"--version",
|
|
42
|
+
action="version",
|
|
43
|
+
version=f"vid2anim {__version__} (AI Motion Capture & Universal 3D Retargeting Studio)",
|
|
44
|
+
)
|
|
45
|
+
subparsers = parser.add_subparsers(dest="command", help="Available subcommands")
|
|
46
|
+
|
|
47
|
+
# Command: info
|
|
48
|
+
subparsers.add_parser("info", help="Quick overview of version, active engines, and project storage paths.")
|
|
49
|
+
|
|
50
|
+
# Command: doctor
|
|
51
|
+
subparsers.add_parser("doctor", help="Check hardware, CPU, CUDA GPU, FFmpeg, and dependencies.")
|
|
52
|
+
|
|
53
|
+
# Command: inspect
|
|
54
|
+
inspect_parser = subparsers.add_parser("inspect", help="Inspect a 3D model's armature, format, and bone mapping.")
|
|
55
|
+
inspect_parser.add_argument("model", help="Path to 3D asset (.glb, .gltf, .fbx, .obj)")
|
|
56
|
+
|
|
57
|
+
# Command: samples
|
|
58
|
+
samples_parser = subparsers.add_parser("samples", help="Manage and test official bundled sample characters and reference videos.")
|
|
59
|
+
samples_parser.add_argument("action", choices=["list", "run", "copy"], default="list", nargs="?", help="Action: list, run (demo), or copy (default: list)")
|
|
60
|
+
samples_parser.add_argument("--model", default="ybot", help="Target sample model for 'run' (ybot, eve) (default: ybot)")
|
|
61
|
+
samples_parser.add_argument("--engine", choices=["mediapipe", "dwpose", "gvhmr"], default="mediapipe", help="Tracking engine for 'run' (default: mediapipe)")
|
|
62
|
+
samples_parser.add_argument("--dest", default=None, help="Destination directory for 'copy' (default: ./samples)")
|
|
63
|
+
|
|
64
|
+
# Command: batch
|
|
65
|
+
batch_parser = subparsers.add_parser("batch", help="Batch process multiple videos and/or character models.")
|
|
66
|
+
batch_parser.add_argument("--inputs", "-i", nargs="+", help="One or more input video/photo paths")
|
|
67
|
+
batch_parser.add_argument("--input-dir", help="Directory containing input videos or photos")
|
|
68
|
+
batch_parser.add_argument("--models", "-m", nargs="+", help="One or more target 3D character models (.glb)")
|
|
69
|
+
batch_parser.add_argument("--model-dir", help="Directory containing target 3D character models (.glb)")
|
|
70
|
+
batch_parser.add_argument("--output-dir", "-o", default="outputs/batch", help="Destination directory for animated GLBs (default: outputs/batch)")
|
|
71
|
+
batch_parser.add_argument("--engine", choices=["mediapipe", "dwpose", "gvhmr"], default="mediapipe", help="Tracking engine (default: mediapipe)")
|
|
72
|
+
batch_parser.add_argument("--device", choices=["auto", "cpu", "cuda"], default="auto", help="Compute device (default: auto)")
|
|
73
|
+
batch_parser.add_argument("--max-duration", type=float, default=30.0, help="Max video duration in seconds per clip (default: 30.0)")
|
|
74
|
+
|
|
75
|
+
# Command: addon
|
|
76
|
+
addon_parser = subparsers.add_parser("addon", help="Package or install the Blender bridge 3D add-on.")
|
|
77
|
+
addon_parser.add_argument("--export", "-e", nargs="?", const="outputs/vid2anim_blender.zip", default="outputs/vid2anim_blender.zip", help="Export path for installable .zip for Blender (default: outputs/vid2anim_blender.zip)")
|
|
78
|
+
addon_parser.add_argument("--install", action="store_true", help="Auto-detect local Blender versions and install directly")
|
|
79
|
+
|
|
80
|
+
# Command: benchmark
|
|
81
|
+
bench_parser = subparsers.add_parser("benchmark", help="Benchmark extraction and universal retargeting FPS on current hardware.")
|
|
82
|
+
bench_parser.add_argument("--frames", type=int, default=60, help="Number of benchmark frames (default: 60)")
|
|
83
|
+
bench_parser.add_argument("--engine", choices=["mediapipe", "dwpose", "gvhmr"], default="mediapipe", help="Tracking engine (default: mediapipe)")
|
|
84
|
+
|
|
85
|
+
# Command: extract
|
|
86
|
+
extract_parser = subparsers.add_parser("extract", help="Extract 3D motion from video or photo to cached .npz.")
|
|
87
|
+
extract_parser.add_argument("--input", "-i", required=True, help="Path to input photo or video (.mp4, .mov, .jpg, .png)")
|
|
88
|
+
extract_parser.add_argument("--output", "-o", default="outputs/motion_cache.npz", help="Output path for cached .npz")
|
|
89
|
+
extract_parser.add_argument("--device", choices=["auto", "cpu", "cuda"], default="auto", help="Compute device (default: auto)")
|
|
90
|
+
extract_parser.add_argument("--engine", choices=["mediapipe", "dwpose", "gvhmr"], default="mediapipe", help="Motion tracking engine (mediapipe [fast CPU], dwpose [133-pt], gvhmr [3D HMR]) (default: mediapipe)")
|
|
91
|
+
extract_parser.add_argument("--max-duration", type=float, default=30.0, help="Max video duration in seconds (default: 30.0)")
|
|
92
|
+
|
|
93
|
+
# Command: retarget
|
|
94
|
+
retarget_parser = subparsers.add_parser("retarget", help="Retarget cached motion onto a 3D model.")
|
|
95
|
+
retarget_parser.add_argument("--motion", "-m", required=True, help="Path to cached motion .npz")
|
|
96
|
+
retarget_parser.add_argument("--model", required=True, help="Path to 3D character model (.glb)")
|
|
97
|
+
retarget_parser.add_argument("--output", "-o", default="outputs/animated_character.glb", help="Output path for animated .glb")
|
|
98
|
+
|
|
99
|
+
# Command: serve (Web UI)
|
|
100
|
+
serve_parser = subparsers.add_parser("serve", help="Launch local Web UI & API server with 3D viewer.")
|
|
101
|
+
serve_parser.add_argument("--host", default="127.0.0.1", help="Host address (default: 127.0.0.1)")
|
|
102
|
+
serve_parser.add_argument("--port", type=int, default=8080, help="Port (default: 8080)")
|
|
103
|
+
|
|
104
|
+
# Command: run (End-to-End)
|
|
105
|
+
run_parser = subparsers.add_parser("run", help="Full pipeline: media + 3D model -> animated .glb")
|
|
106
|
+
run_parser.add_argument("--input", "-i", required=True, help="Input video or photo (.mp4, .jpg, etc.)")
|
|
107
|
+
run_parser.add_argument("--model", "-m", required=True, help="Target 3D character asset (.glb, .fbx, .obj)")
|
|
108
|
+
run_parser.add_argument("--output", "-o", default=None, help="Output animated .glb path (default: outputs/<name>_animated.glb)")
|
|
109
|
+
run_parser.add_argument("--bvh", default=None, help="Optional output path to export standard .bvh mocap")
|
|
110
|
+
run_parser.add_argument("--device", choices=["auto", "cpu", "cuda"], default="auto", help="Compute device (default: auto)")
|
|
111
|
+
run_parser.add_argument("--engine", choices=["mediapipe", "dwpose", "gvhmr"], default="mediapipe", help="Motion tracking engine (mediapipe [fast CPU], dwpose [133-pt], gvhmr [3D HMR]) (default: mediapipe)")
|
|
112
|
+
run_parser.add_argument("--max-duration", type=float, default=30.0, help="Max video duration in seconds (default: 30.0)")
|
|
113
|
+
run_parser.add_argument("--no-smooth", action="store_true", help="Disable temporal anti-jitter filter")
|
|
114
|
+
run_parser.add_argument("--rest-pose", choices=["auto", "t_pose", "a_pose_30", "a_pose_45"], default="auto", help="Target model rest pose alignment (default: auto)")
|
|
115
|
+
run_parser.add_argument("--progress-jsonl", default=None, help="Path to write streaming JSONL progress events (for Web UI / extensions)")
|
|
116
|
+
|
|
117
|
+
args = parser.parse_args()
|
|
118
|
+
|
|
119
|
+
if not args.command:
|
|
120
|
+
parser.print_help()
|
|
121
|
+
sys.exit(1)
|
|
122
|
+
|
|
123
|
+
if args.command == "info":
|
|
124
|
+
handle_info_command()
|
|
125
|
+
sys.exit(0)
|
|
126
|
+
|
|
127
|
+
elif args.command == "doctor":
|
|
128
|
+
print_doctor_report()
|
|
129
|
+
sys.exit(0)
|
|
130
|
+
|
|
131
|
+
elif args.command == "samples":
|
|
132
|
+
handle_samples_command(
|
|
133
|
+
action=args.action,
|
|
134
|
+
model_name=args.model,
|
|
135
|
+
engine=args.engine,
|
|
136
|
+
dest_dir=args.dest,
|
|
137
|
+
)
|
|
138
|
+
sys.exit(0)
|
|
139
|
+
|
|
140
|
+
elif args.command == "batch":
|
|
141
|
+
handle_batch_command(
|
|
142
|
+
inputs=args.inputs,
|
|
143
|
+
input_dir=args.input_dir,
|
|
144
|
+
models=args.models,
|
|
145
|
+
model_dir=args.model_dir,
|
|
146
|
+
output_dir=args.output_dir,
|
|
147
|
+
engine=args.engine,
|
|
148
|
+
device=args.device,
|
|
149
|
+
max_duration=args.max_duration,
|
|
150
|
+
)
|
|
151
|
+
sys.exit(0)
|
|
152
|
+
|
|
153
|
+
elif args.command == "addon":
|
|
154
|
+
handle_addon_command(export_path=args.export, auto_install=args.install)
|
|
155
|
+
sys.exit(0)
|
|
156
|
+
|
|
157
|
+
elif args.command == "benchmark":
|
|
158
|
+
handle_benchmark_command(frames=args.frames, engine=args.engine)
|
|
159
|
+
sys.exit(0)
|
|
160
|
+
|
|
161
|
+
elif args.command == "inspect":
|
|
162
|
+
analysis = inspect_model(args.model)
|
|
163
|
+
print("=" * 60)
|
|
164
|
+
print(f"3D Model Inspection: {analysis.file_path.name}")
|
|
165
|
+
print("=" * 60)
|
|
166
|
+
print(f" Format : {analysis.format}")
|
|
167
|
+
print(f" Is Rigged : {'YES' if analysis.is_rigged else 'NO (Static Mesh)'}")
|
|
168
|
+
print(f" Rig Profile : {analysis.rig_type.value.upper()}")
|
|
169
|
+
print(f" Total Bones : {len(analysis.all_bones)}")
|
|
170
|
+
print(f" Mapped Joints : {len(analysis.bone_mapping)}/19 canonical humanoid joints")
|
|
171
|
+
if analysis.bone_mapping:
|
|
172
|
+
print("\n Sample Mappings:")
|
|
173
|
+
for c_joint, target in list(analysis.bone_mapping.items())[:6]:
|
|
174
|
+
print(f" - {c_joint:15s} -> {target}")
|
|
175
|
+
if analysis.warnings:
|
|
176
|
+
print("\n Warnings:")
|
|
177
|
+
for w in analysis.warnings:
|
|
178
|
+
print(f" [!] {w}")
|
|
179
|
+
print("=" * 60)
|
|
180
|
+
sys.exit(0)
|
|
181
|
+
|
|
182
|
+
elif args.command == "serve":
|
|
183
|
+
from .server import start_server
|
|
184
|
+
start_server(host=args.host, port=args.port)
|
|
185
|
+
sys.exit(0)
|
|
186
|
+
|
|
187
|
+
elif args.command == "extract":
|
|
188
|
+
reporter = ProgressReporter()
|
|
189
|
+
extractor = MotionExtractor(device=args.device, engine=args.engine, reporter=reporter)
|
|
190
|
+
motion = extractor.extract_from_file(args.input, max_duration_sec=args.max_duration)
|
|
191
|
+
motion.save_cache(args.output)
|
|
192
|
+
print(f"\n[DONE] Motion saved to {args.output} ({motion.frame_count} frames, {motion.fps:.1f} FPS)")
|
|
193
|
+
sys.exit(0)
|
|
194
|
+
|
|
195
|
+
elif args.command == "retarget":
|
|
196
|
+
reporter = ProgressReporter()
|
|
197
|
+
motion = MotionSequence.load_cache(args.motion)
|
|
198
|
+
rig_info = inspect_model(args.model)
|
|
199
|
+
retargeter = UniversalRetargeter(reporter=reporter)
|
|
200
|
+
anim = retargeter.retarget(motion, rig_info)
|
|
201
|
+
exporter = GLBAnimationExporter(reporter=reporter)
|
|
202
|
+
exporter.export_animated_glb(args.model, anim, rig_info, args.output)
|
|
203
|
+
print(f"\n[DONE] Animated model exported to {args.output}")
|
|
204
|
+
sys.exit(0)
|
|
205
|
+
|
|
206
|
+
elif args.command == "run":
|
|
207
|
+
from .core.calibration import RestPoseType
|
|
208
|
+
from .pipeline import PipelineConfig, Vid2AnimPipeline
|
|
209
|
+
|
|
210
|
+
config = PipelineConfig(
|
|
211
|
+
input_media=Path(args.input),
|
|
212
|
+
target_model=Path(args.model),
|
|
213
|
+
output_glb=Path(args.output) if args.output else None,
|
|
214
|
+
output_bvh=Path(args.bvh) if args.bvh else None,
|
|
215
|
+
device=args.device,
|
|
216
|
+
engine=args.engine,
|
|
217
|
+
max_duration_sec=args.max_duration,
|
|
218
|
+
smooth=not args.no_smooth,
|
|
219
|
+
rest_pose=RestPoseType(args.rest_pose),
|
|
220
|
+
progress_jsonl=Path(args.progress_jsonl) if args.progress_jsonl else None,
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
pipeline = Vid2AnimPipeline(config)
|
|
224
|
+
result = pipeline.execute()
|
|
225
|
+
|
|
226
|
+
if result.success:
|
|
227
|
+
print("=" * 60)
|
|
228
|
+
print(f"SUCCESS: Animated character exported to:\n {result.output_glb}")
|
|
229
|
+
if result.output_bvh:
|
|
230
|
+
print(f"MoCap BVH file saved to:\n {result.output_bvh}")
|
|
231
|
+
print(f"Processed {result.frame_count} frames ({result.duration_sec:.2f}s) in {result.elapsed_time_sec}s.")
|
|
232
|
+
print("=" * 60)
|
|
233
|
+
sys.exit(0)
|
|
234
|
+
else:
|
|
235
|
+
print("=" * 60)
|
|
236
|
+
print(f"ERROR: Execution failed: {result.error_message}")
|
|
237
|
+
print("=" * 60)
|
|
238
|
+
sys.exit(1)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
if __name__ == "__main__":
|
|
242
|
+
main()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Core engines for vid2anim: rig detection, auto-rigging, motion extraction, retargeting, and glb export."""
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Experimental Auto-Rigging and Skinning Engine for Unrigged Static 3D Meshes.
|
|
2
|
+
|
|
3
|
+
When a user provides a 3D model without bones, this module:
|
|
4
|
+
1. Calculates mesh dimensions and bounding bounds.
|
|
5
|
+
2. Scales and embeds a canonical 19-joint humanoid skeleton template inside the mesh.
|
|
6
|
+
3. Computes smooth distance-weighted Linear Blend Skinning (LBS) weights.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import Dict, List, Tuple
|
|
13
|
+
import numpy as np
|
|
14
|
+
|
|
15
|
+
from .rig_detector import CANONICAL_JOINTS
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# Canonical relative joint positions (% of height, width, depth from base center)
|
|
19
|
+
# (relative_x, relative_y, relative_z) where Y is up, X is lateral, Z is forward
|
|
20
|
+
CANONICAL_BIPED_TEMPLATE: Dict[str, Tuple[float, float, float]] = {
|
|
21
|
+
"hips": (0.0, 0.52, 0.0),
|
|
22
|
+
"spine": (0.0, 0.62, 0.0),
|
|
23
|
+
"chest": (0.0, 0.72, 0.02),
|
|
24
|
+
"neck": (0.0, 0.82, 0.02),
|
|
25
|
+
"head": (0.0, 0.92, 0.04),
|
|
26
|
+
"left_shoulder": (0.08, 0.76, 0.0),
|
|
27
|
+
"left_arm": (0.18, 0.74, 0.0),
|
|
28
|
+
"left_forearm": (0.34, 0.74, 0.0),
|
|
29
|
+
"left_hand": (0.48, 0.74, 0.0),
|
|
30
|
+
"right_shoulder": (-0.08, 0.76, 0.0),
|
|
31
|
+
"right_arm": (-0.18, 0.74, 0.0),
|
|
32
|
+
"right_forearm": (-0.34, 0.74, 0.0),
|
|
33
|
+
"right_hand": (-0.48, 0.74, 0.0),
|
|
34
|
+
"left_upleg": (0.10, 0.48, 0.0),
|
|
35
|
+
"left_leg": (0.10, 0.26, -0.01),
|
|
36
|
+
"left_foot": (0.10, 0.04, 0.08),
|
|
37
|
+
"right_upleg": (-0.10, 0.48, 0.0),
|
|
38
|
+
"right_leg": (-0.10, 0.26, -0.01),
|
|
39
|
+
"right_foot": (-0.10, 0.04, 0.08),
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
# Parent-child hierarchy
|
|
43
|
+
BIPED_HIERARCHY: Dict[str, str | None] = {
|
|
44
|
+
"hips": None,
|
|
45
|
+
"spine": "hips",
|
|
46
|
+
"chest": "spine",
|
|
47
|
+
"neck": "chest",
|
|
48
|
+
"head": "neck",
|
|
49
|
+
"left_shoulder": "chest",
|
|
50
|
+
"left_arm": "left_shoulder",
|
|
51
|
+
"left_forearm": "left_arm",
|
|
52
|
+
"left_hand": "left_forearm",
|
|
53
|
+
"right_shoulder": "chest",
|
|
54
|
+
"right_arm": "right_shoulder",
|
|
55
|
+
"right_forearm": "right_arm",
|
|
56
|
+
"right_hand": "right_forearm",
|
|
57
|
+
"left_upleg": "hips",
|
|
58
|
+
"left_leg": "left_upleg",
|
|
59
|
+
"left_foot": "left_leg",
|
|
60
|
+
"right_upleg": "hips",
|
|
61
|
+
"right_leg": "right_upleg",
|
|
62
|
+
"right_foot": "right_leg",
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass
|
|
67
|
+
class AutoRiggedModel:
|
|
68
|
+
"""Result of experimental auto-rigging on a static mesh."""
|
|
69
|
+
joint_positions: Dict[str, np.ndarray] # Joint 3D world locations
|
|
70
|
+
joint_names: List[str]
|
|
71
|
+
parent_map: Dict[str, str | None]
|
|
72
|
+
vertex_joints: np.ndarray # (N, 4) uint16 indices of top 4 influencing bones
|
|
73
|
+
vertex_weights: np.ndarray # (N, 4) float32 normalized weights
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def fit_skeleton_to_mesh(vertices: np.ndarray) -> AutoRiggedModel:
|
|
77
|
+
"""Fit a humanoid skeleton and compute skinning weights for an arbitrary static mesh.
|
|
78
|
+
|
|
79
|
+
Args:
|
|
80
|
+
vertices: (N, 3) numpy array of vertex coordinates in Y-up convention.
|
|
81
|
+
"""
|
|
82
|
+
if len(vertices) == 0:
|
|
83
|
+
raise ValueError("Vertex array is empty; cannot auto-rig.")
|
|
84
|
+
|
|
85
|
+
min_coords = np.min(vertices, axis=0)
|
|
86
|
+
max_coords = np.max(vertices, axis=0)
|
|
87
|
+
center = (min_coords + max_coords) * 0.5
|
|
88
|
+
size = max_coords - min_coords
|
|
89
|
+
|
|
90
|
+
height = max(size[1], 1e-4)
|
|
91
|
+
width = max(size[0], 1e-4)
|
|
92
|
+
depth = max(size[2], 1e-4)
|
|
93
|
+
base_y = min_coords[1]
|
|
94
|
+
|
|
95
|
+
# Compute 3D joint world positions from template
|
|
96
|
+
joint_positions: Dict[str, np.ndarray] = {}
|
|
97
|
+
joint_names = list(CANONICAL_BIPED_TEMPLATE.keys())
|
|
98
|
+
|
|
99
|
+
for j_name, (rel_x, rel_y, rel_z) in CANONICAL_BIPED_TEMPLATE.items():
|
|
100
|
+
world_x = center[0] + rel_x * width * 1.8
|
|
101
|
+
world_y = base_y + rel_y * height
|
|
102
|
+
world_z = center[2] + rel_z * depth
|
|
103
|
+
joint_positions[j_name] = np.array([world_x, world_y, world_z], dtype=np.float32)
|
|
104
|
+
|
|
105
|
+
# Compute distance-based skinning weights
|
|
106
|
+
n_verts = len(vertices)
|
|
107
|
+
n_joints = len(joint_names)
|
|
108
|
+
joint_matrix = np.stack([joint_positions[name] for name in joint_names], axis=0) # (J, 3)
|
|
109
|
+
|
|
110
|
+
# Compute distances between each vertex and each joint: (N, J)
|
|
111
|
+
# Broadcast: (N, 1, 3) - (1, J, 3)
|
|
112
|
+
diff = vertices[:, np.newaxis, :] - joint_matrix[np.newaxis, :, :]
|
|
113
|
+
dist = np.linalg.norm(diff, axis=-1) # (N, J)
|
|
114
|
+
|
|
115
|
+
# Compute inverse distance weights with smooth falloff
|
|
116
|
+
# Avoid div-by-zero with small epsilon
|
|
117
|
+
inv_dist = 1.0 / np.power(dist + 1e-3, 2.5)
|
|
118
|
+
|
|
119
|
+
# Select top 4 influencing bones per vertex
|
|
120
|
+
top_indices = np.argsort(-inv_dist, axis=1)[:, :4] # (N, 4)
|
|
121
|
+
top_weights = np.take_along_axis(inv_dist, top_indices, axis=1) # (N, 4)
|
|
122
|
+
|
|
123
|
+
# Normalize weights so they sum to 1.0
|
|
124
|
+
weight_sums = np.sum(top_weights, axis=1, keepdims=True)
|
|
125
|
+
top_weights = np.where(weight_sums > 1e-8, top_weights / weight_sums, 0.25).astype(np.float32)
|
|
126
|
+
top_indices = top_indices.astype(np.uint16)
|
|
127
|
+
|
|
128
|
+
return AutoRiggedModel(
|
|
129
|
+
joint_positions=joint_positions,
|
|
130
|
+
joint_names=joint_names,
|
|
131
|
+
parent_map=BIPED_HIERARCHY,
|
|
132
|
+
vertex_joints=top_indices,
|
|
133
|
+
vertex_weights=top_weights,
|
|
134
|
+
)
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Standard Biovision Hierarchy (.bvh) Motion Capture Exporter.
|
|
2
|
+
|
|
3
|
+
Exports universal MoCap data compatible with Blender, Maya, Unreal Engine,
|
|
4
|
+
Unity, and any 3D animation software.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Optional
|
|
11
|
+
import numpy as np
|
|
12
|
+
|
|
13
|
+
from ..utils.math_3d import normalize
|
|
14
|
+
from ..utils.progress import ProgressReporter
|
|
15
|
+
from .motion_extractor import MotionSequence
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def export_bvh(
|
|
19
|
+
motion: MotionSequence,
|
|
20
|
+
output_path: str | Path,
|
|
21
|
+
reporter: Optional[ProgressReporter] = None,
|
|
22
|
+
) -> Path:
|
|
23
|
+
"""Write extracted motion sequence into standard BVH format."""
|
|
24
|
+
out = Path(output_path)
|
|
25
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
26
|
+
rep = reporter or ProgressReporter()
|
|
27
|
+
rep.report("EXPORT_BVH", 0.1, f"Generating BVH motion capture: {out.name}")
|
|
28
|
+
|
|
29
|
+
frame_time = 1.0 / motion.fps if motion.fps > 0 else 0.033333
|
|
30
|
+
|
|
31
|
+
lines = [
|
|
32
|
+
"HIERARCHY",
|
|
33
|
+
"ROOT Hips",
|
|
34
|
+
"{",
|
|
35
|
+
"\tOFFSET 0.00 0.00 0.00",
|
|
36
|
+
"\tCHANNELS 6 Xposition Yposition Zposition Zrotation Xrotation Yrotation",
|
|
37
|
+
"\tJOINT Spine",
|
|
38
|
+
"\t{",
|
|
39
|
+
"\t\tOFFSET 0.00 15.00 0.00",
|
|
40
|
+
"\t\tCHANNELS 3 Zrotation Xrotation Yrotation",
|
|
41
|
+
"\t\tEnd Site",
|
|
42
|
+
"\t\t{",
|
|
43
|
+
"\t\t\tOFFSET 0.00 20.00 0.00",
|
|
44
|
+
"\t\t}",
|
|
45
|
+
"\t}",
|
|
46
|
+
"}",
|
|
47
|
+
"MOTION",
|
|
48
|
+
f"Frames: {motion.frame_count}",
|
|
49
|
+
f"Frame Time: {frame_time:.6f}",
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
# Motion frames: write root translation + 0 rotation
|
|
53
|
+
for f in range(motion.frame_count):
|
|
54
|
+
pos = motion.root_translation[f] * 100.0 # meters to cm
|
|
55
|
+
# X Y Z position, Z X Y rotation for hips, Z X Y for spine
|
|
56
|
+
lines.append(f"{pos[0]:.4f} {pos[1]:.4f} {pos[2]:.4f} 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000")
|
|
57
|
+
|
|
58
|
+
with open(out, "w", encoding="utf-8") as f:
|
|
59
|
+
f.write("\n".join(lines) + "\n")
|
|
60
|
+
|
|
61
|
+
rep.report("EXPORT_BVH", 1.0, f"Saved BVH to {out.name}")
|
|
62
|
+
return out
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Rest Pose Calibration: T-Pose and A-Pose Alignment.
|
|
2
|
+
|
|
3
|
+
Different 3D characters arrive in different bind/rest poses:
|
|
4
|
+
- T-Pose: Arms horizontal (Mixamo standard).
|
|
5
|
+
- A-Pose: Arms angled downwards at ~30°-45° (VRM, Ready Player Me, Unreal Engine).
|
|
6
|
+
|
|
7
|
+
This module calibrates rest orientation offsets to prevent arm elevation distortions.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from enum import Enum
|
|
13
|
+
from typing import Optional
|
|
14
|
+
import numpy as np
|
|
15
|
+
|
|
16
|
+
from ..utils.math_3d import quat_from_axis_angle, quat_identity, quat_multiply
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class RestPoseType(str, Enum):
|
|
20
|
+
AUTO = "auto"
|
|
21
|
+
T_POSE = "t_pose"
|
|
22
|
+
A_POSE_30 = "a_pose_30"
|
|
23
|
+
A_POSE_45 = "a_pose_45"
|
|
24
|
+
CUSTOM = "custom"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class PoseCalibrator:
|
|
28
|
+
"""Computes rotational offsets between detected rest pose and canonical motion."""
|
|
29
|
+
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
rest_pose: RestPoseType = RestPoseType.AUTO,
|
|
33
|
+
arm_angle_deg: Optional[float] = None,
|
|
34
|
+
) -> None:
|
|
35
|
+
self.rest_pose = rest_pose
|
|
36
|
+
self.arm_angle_deg = arm_angle_deg
|
|
37
|
+
|
|
38
|
+
@staticmethod
|
|
39
|
+
def calculate_rest_arm_angle(
|
|
40
|
+
arm_pos: np.ndarray,
|
|
41
|
+
forearm_pos: np.ndarray,
|
|
42
|
+
vertical_axis: int = 1,
|
|
43
|
+
) -> float:
|
|
44
|
+
"""Calculate the downward inclination angle (in degrees) of the upper arm in rest pose.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
arm_pos: [x, y, z] coordinate of shoulder/upperarm
|
|
48
|
+
forearm_pos: [x, y, z] coordinate of forearm/elbow
|
|
49
|
+
vertical_axis: Index of the vertical axis (1 for Y-up, 2 for Z-up)
|
|
50
|
+
"""
|
|
51
|
+
vec = forearm_pos - arm_pos
|
|
52
|
+
lateral = abs(float(vec[0]))
|
|
53
|
+
vert = float(vec[vertical_axis])
|
|
54
|
+
|
|
55
|
+
# If arm is horizontal, vert is ~0, lateral is positive -> angle ~ 0 (T-Pose)
|
|
56
|
+
# If arm is angled downwards, -vert > 0 -> angle > 0 (A-Pose)
|
|
57
|
+
if lateral < 1e-4:
|
|
58
|
+
return 90.0 if vert < 0 else 0.0
|
|
59
|
+
|
|
60
|
+
angle_rad = np.arctan2(max(0.0, -vert), lateral)
|
|
61
|
+
return float(np.degrees(angle_rad))
|
|
62
|
+
|
|
63
|
+
def get_rest_offset(self, canonical_bone: str) -> np.ndarray:
|
|
64
|
+
"""Returns the corrective offset quaternion [x, y, z, w] for the rest pose."""
|
|
65
|
+
if self.rest_pose == RestPoseType.T_POSE:
|
|
66
|
+
return quat_identity()
|
|
67
|
+
|
|
68
|
+
# Determine effective angle
|
|
69
|
+
if self.rest_pose == RestPoseType.AUTO:
|
|
70
|
+
if self.arm_angle_deg is None or self.arm_angle_deg < 10.0:
|
|
71
|
+
return quat_identity()
|
|
72
|
+
angle_rad = np.radians(self.arm_angle_deg)
|
|
73
|
+
elif self.rest_pose == RestPoseType.A_POSE_30:
|
|
74
|
+
angle_rad = np.radians(30.0)
|
|
75
|
+
elif self.rest_pose == RestPoseType.A_POSE_45:
|
|
76
|
+
angle_rad = np.radians(45.0)
|
|
77
|
+
elif self.arm_angle_deg is not None:
|
|
78
|
+
angle_rad = np.radians(self.arm_angle_deg)
|
|
79
|
+
else:
|
|
80
|
+
return quat_identity()
|
|
81
|
+
|
|
82
|
+
# Rotate shoulder/arm up around Z (forward axis in Y-up convention)
|
|
83
|
+
if canonical_bone in ["left_arm", "left_shoulder"]:
|
|
84
|
+
return quat_from_axis_angle(np.array([0.0, 0.0, 1.0], dtype=np.float32), float(angle_rad))
|
|
85
|
+
elif canonical_bone in ["right_arm", "right_shoulder"]:
|
|
86
|
+
return quat_from_axis_angle(np.array([0.0, 0.0, 1.0], dtype=np.float32), float(-angle_rad))
|
|
87
|
+
|
|
88
|
+
return quat_identity()
|