smplfitter 0.3.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. smplfitter/__init__.py +21 -0
  2. smplfitter/_version.py +34 -0
  3. smplfitter/common.py +266 -0
  4. smplfitter/decimation/__init__.py +1 -0
  5. smplfitter/decimation/decimate_body_models.py +55 -0
  6. smplfitter/decimation/make_post_lbs_joint_regressors.py +290 -0
  7. smplfitter/decimation/make_post_lbs_joint_regressors_tf.py +272 -0
  8. smplfitter/download.py +412 -0
  9. smplfitter/jax/__init__.py +23 -0
  10. smplfitter/jax/bodyconverter.py +156 -0
  11. smplfitter/jax/bodyfitter.py +1004 -0
  12. smplfitter/jax/bodymodel.py +200 -0
  13. smplfitter/jax/lstsq.py +119 -0
  14. smplfitter/jax/rotation.py +98 -0
  15. smplfitter/nb/__init__.py +23 -0
  16. smplfitter/nb/bodyconverter.py +153 -0
  17. smplfitter/nb/bodyfitter.py +1321 -0
  18. smplfitter/nb/bodymodel.py +454 -0
  19. smplfitter/nb/lstsq.py +315 -0
  20. smplfitter/nb/precompile.py +109 -0
  21. smplfitter/nb/rotation.py +120 -0
  22. smplfitter/nb/util.py +58 -0
  23. smplfitter/np/__init__.py +22 -0
  24. smplfitter/np/bodyconverter.py +153 -0
  25. smplfitter/np/bodyfitter.py +966 -0
  26. smplfitter/np/bodymodel.py +318 -0
  27. smplfitter/np/lstsq.py +69 -0
  28. smplfitter/np/rotation.py +78 -0
  29. smplfitter/np/util.py +18 -0
  30. smplfitter/pt/__init__.py +156 -0
  31. smplfitter/pt/bodyconverter.py +170 -0
  32. smplfitter/pt/bodyfitter.py +1093 -0
  33. smplfitter/pt/bodyfitter_opt.py +255 -0
  34. smplfitter/pt/bodyflipper.py +169 -0
  35. smplfitter/pt/bodyflipper_opt.py +181 -0
  36. smplfitter/pt/bodymodel.py +408 -0
  37. smplfitter/pt/handreplacer.py +79 -0
  38. smplfitter/pt/lstsq.py +90 -0
  39. smplfitter/pt/rotation.py +73 -0
  40. smplfitter/tf/__init__.py +186 -0
  41. smplfitter/tf/bodyconverter.py +168 -0
  42. smplfitter/tf/bodyfitter.py +1074 -0
  43. smplfitter/tf/bodymodel.py +333 -0
  44. smplfitter/tf/lstsq.py +91 -0
  45. smplfitter/tf/rotation.py +66 -0
  46. smplfitter/tf/util.py +12 -0
  47. smplfitter-0.3.2.dist-info/METADATA +292 -0
  48. smplfitter-0.3.2.dist-info/RECORD +51 -0
  49. smplfitter-0.3.2.dist-info/WHEEL +5 -0
  50. smplfitter-0.3.2.dist-info/licenses/LICENSE +21 -0
  51. smplfitter-0.3.2.dist-info/top_level.txt +1 -0
smplfitter/__init__.py ADDED
@@ -0,0 +1,21 @@
1
+ """SMPLFitter provides forward and inverse kinematics for SMPL-family body models.
2
+
3
+ Main submodules:
4
+ - :mod:`smplfitter.np` - NumPy backend
5
+ - :mod:`smplfitter.pt` - PyTorch backend
6
+ - :mod:`smplfitter.tf` - TensorFlow backend
7
+ - :mod:`smplfitter.jax` - JAX backend
8
+ - :mod:`smplfitter.nb` - Numba backend
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from .common import ModelData, initialize, _set_module_for_docs
14
+
15
+ try:
16
+ from ._version import version as __version__
17
+ except ImportError:
18
+ __version__ = '0.0.0'
19
+
20
+ __all__ = ['ModelData', 'initialize', '__version__']
21
+ _set_module_for_docs(__name__, globals(), __all__)
smplfitter/_version.py ADDED
@@ -0,0 +1,34 @@
1
+ # file generated by setuptools-scm
2
+ # don't change, don't track in version control
3
+
4
+ __all__ = [
5
+ "__version__",
6
+ "__version_tuple__",
7
+ "version",
8
+ "version_tuple",
9
+ "__commit_id__",
10
+ "commit_id",
11
+ ]
12
+
13
+ TYPE_CHECKING = False
14
+ if TYPE_CHECKING:
15
+ from typing import Tuple
16
+ from typing import Union
17
+
18
+ VERSION_TUPLE = Tuple[Union[int, str], ...]
19
+ COMMIT_ID = Union[str, None]
20
+ else:
21
+ VERSION_TUPLE = object
22
+ COMMIT_ID = object
23
+
24
+ version: str
25
+ __version__: str
26
+ __version_tuple__: VERSION_TUPLE
27
+ version_tuple: VERSION_TUPLE
28
+ commit_id: COMMIT_ID
29
+ __commit_id__: COMMIT_ID
30
+
31
+ __version__ = version = '0.3.2'
32
+ __version_tuple__ = version_tuple = (0, 3, 2)
33
+
34
+ __commit_id__ = commit_id = None
smplfitter/common.py ADDED
@@ -0,0 +1,266 @@
1
+ from __future__ import annotations
2
+
3
+ import contextlib
4
+ import os
5
+ import os.path as osp
6
+ import pickle
7
+ import sys
8
+ import warnings
9
+ from dataclasses import dataclass
10
+
11
+ import numpy as np
12
+
13
+
14
+ def _set_module_for_docs(module_name, module_globals, all_names):
15
+ """Override __module__ on exported objects so Sphinx resolves package-level names.
16
+
17
+ sphinx-codeautolink uses __module__ to find the docs page for a name. Without this,
18
+ e.g. ``BodyModel`` imported from ``smplfitter.pt.bodymodel`` would not link to
19
+ ``smplfitter.pt.BodyModel``. The original __module__ is saved as ``_module_original_``
20
+ so that ``inspect.getsourcefile`` can still find the real source file (see
21
+ ``module_restored`` in docs/conf.py).
22
+ """
23
+ for name in all_names:
24
+ obj = module_globals.get(name)
25
+ if obj is not None and callable(obj):
26
+ obj._module_original_ = obj.__module__
27
+ obj.__module__ = module_name
28
+
29
+
30
+ @dataclass
31
+ class ModelData:
32
+ """Data loaded from a SMPL-family body model file.
33
+
34
+ This dataclass holds all arrays and metadata needed to instantiate a body model
35
+ in any backend (NumPy, PyTorch, TensorFlow, JAX, Numba).
36
+ """
37
+
38
+ # Tensor data (numpy arrays, to be converted to framework-specific tensors)
39
+ v_template: np.ndarray
40
+ """Vertex template in T-pose, shape (num_vertices, 3)."""
41
+
42
+ shapedirs: np.ndarray
43
+ """Shape blend shapes, shape (num_vertices, 3, num_betas)."""
44
+
45
+ posedirs: np.ndarray
46
+ """Pose blend shapes, shape (num_vertices, 3, (num_joints-1)*9)."""
47
+
48
+ J_regressor_post_lbs: np.ndarray
49
+ """Joint regressor for post-LBS joint locations, shape (num_joints, num_vertices)."""
50
+
51
+ J_template: np.ndarray
52
+ """Joint template positions, shape (num_joints, 3)."""
53
+
54
+ J_shapedirs: np.ndarray
55
+ """Joint shape directions, shape (num_joints, 3, num_betas)."""
56
+
57
+ kid_shapedir: np.ndarray
58
+ """Kid shape blend shape for vertices, shape (num_vertices, 3)."""
59
+
60
+ kid_J_shapedir: np.ndarray
61
+ """Kid shape blend shape for joints, shape (num_joints, 3)."""
62
+
63
+ weights: np.ndarray
64
+ """Skinning weights, shape (num_vertices, num_joints)."""
65
+
66
+ # Non-tensor data (metadata)
67
+ kintree_parents: list[int]
68
+ """Parent joint indices for kinematic tree."""
69
+
70
+ faces: np.ndarray
71
+ """Face indices, shape (num_faces, 3)."""
72
+
73
+ num_joints: int
74
+ """Number of joints in the body model."""
75
+
76
+ num_vertices: int
77
+ """Number of vertices in the body model mesh."""
78
+
79
+ vertex_subset: np.ndarray
80
+ """Indices of vertices used (for partial models)."""
81
+
82
+
83
+ def initialize(
84
+ model_name,
85
+ gender,
86
+ model_root=None,
87
+ num_betas=None,
88
+ vertex_subset_size=None,
89
+ vertex_subset=None,
90
+ faces=None,
91
+ joint_regressor_post_lbs=None,
92
+ ):
93
+ if model_root is None:
94
+ body_models_dir = os.getenv('SMPLFITTER_BODY_MODELS')
95
+ if body_models_dir is None:
96
+ data_root = os.getenv('DATA_ROOT', '.')
97
+ body_models_dir = f'{data_root}/body_models'
98
+ model_root = f'{body_models_dir}/{model_name}'
99
+
100
+ with monkey_patched_for_chumpy():
101
+ if model_name == 'smpl':
102
+ gender_str = dict(f='f', m='m', n='neutral')[gender[0]]
103
+ filename = f'basicmodel_{gender_str}_lbs_10_207_0_v1.1.0.pkl'
104
+ elif model_name in ('smplx', 'smplxlh', 'smplxmoyo'):
105
+ gender_str = dict(f='FEMALE', m='MALE', n='NEUTRAL')[gender[0]]
106
+ filename = f'SMPLX_{gender_str}.npz'
107
+ elif model_name == 'smplh':
108
+ gender_str = dict(f='female', m='male')[gender[0]]
109
+ filename = f'SMPLH_{gender_str}.pkl'
110
+ elif model_name == 'smplh16':
111
+ gender_str = dict(f='female', m='male', n='neutral')[gender[0]]
112
+ filename = osp.join(gender_str, 'model.npz')
113
+ elif model_name == 'mano':
114
+ filename = 'MANO_RIGHT.pkl'
115
+ else:
116
+ raise ValueError(f'Unknown model name: {model_name}')
117
+
118
+ filepath = osp.join(model_root, filename)
119
+ try:
120
+ if filename.endswith('.npz'):
121
+ smpl_data = np.load(filepath)
122
+ else:
123
+ with open(filepath, 'rb') as f:
124
+ smpl_data = pickle.load(f, encoding='latin1')
125
+ except FileNotFoundError:
126
+ raise FileNotFoundError(
127
+ f'Body model file not found: {filepath}\n\n'
128
+ f'Set the body model location using one of:\n'
129
+ f" 1. BodyModel('{model_name}', '{gender}', "
130
+ f"model_root='/your/path/body_models/{model_name}')\n"
131
+ f' 2. export SMPLFITTER_BODY_MODELS=/your/path/body_models\n'
132
+ f' 3. export DATA_ROOT=/your/path '
133
+ f'(looks for $DATA_ROOT/body_models/)\n\n'
134
+ f'Download models: python -m smplfitter.download\n'
135
+ f'Register first at https://smpl.is.tue.mpg.de/'
136
+ ) from None
137
+
138
+ res = {}
139
+ res['shapedirs'] = np.array(smpl_data['shapedirs'], dtype=np.float64)
140
+ res['posedirs'] = np.array(smpl_data['posedirs'], dtype=np.float64)
141
+ res['v_template'] = np.array(smpl_data['v_template'], dtype=np.float64)
142
+
143
+ if not isinstance(smpl_data['J_regressor'], np.ndarray):
144
+ res['J_regressor'] = np.array(smpl_data['J_regressor'].toarray(), dtype=np.float64)
145
+ else:
146
+ res['J_regressor'] = smpl_data['J_regressor'].astype(np.float64)
147
+
148
+ res['weights'] = np.array(smpl_data['weights'])
149
+ res['faces'] = np.array(smpl_data['f'].astype(np.int32))
150
+ res['kintree_parents'] = smpl_data['kintree_table'][0].tolist()
151
+ res['num_joints'] = len(res['kintree_parents'])
152
+ res['num_vertices'] = len(res['v_template'])
153
+
154
+ # Kid model has an additional shape parameter which pulls the mesh towards the SMIL mean
155
+ # template
156
+ if model_name.lower().startswith('smpl'):
157
+ kid_path = os.path.join(model_root, 'kid_template.npy')
158
+ try:
159
+ v_template_smil = np.load(kid_path).astype(np.float64)
160
+ except FileNotFoundError:
161
+ raise FileNotFoundError(
162
+ f'Kid template not found: {kid_path}\n'
163
+ f'Download it: python -m smplfitter.download'
164
+ ) from None
165
+ res['kid_shapedir'] = (
166
+ v_template_smil - np.mean(v_template_smil, axis=0) - res['v_template']
167
+ )
168
+ res['kid_J_shapedir'] = res['J_regressor'] @ res['kid_shapedir']
169
+ else:
170
+ res['kid_shapedir'] = np.zeros_like(res['v_template'])
171
+ res['kid_J_shapedir'] = np.zeros((res['num_joints'], 3))
172
+
173
+ if 'J_shapedirs' in smpl_data:
174
+ res['J_shapedirs'] = np.array(smpl_data['J_shapedirs'], dtype=np.float64)
175
+ else:
176
+ res['J_shapedirs'] = np.einsum('jv,vcs->jcs', res['J_regressor'], res['shapedirs'])
177
+
178
+ if 'J_template' in smpl_data:
179
+ res['J_template'] = np.array(smpl_data['J_template'], dtype=np.float64)
180
+ else:
181
+ res['J_template'] = res['J_regressor'] @ res['v_template']
182
+
183
+ res['v_template'] = res['v_template'] - np.einsum(
184
+ 'vcx,x->vc',
185
+ res['posedirs'],
186
+ np.reshape(np.tile(np.eye(3, dtype=np.float64), [res['num_joints'] - 1, 1]), [-1]),
187
+ )
188
+
189
+ if vertex_subset_size is not None:
190
+ vertex_subset_dict = np.load(f'{model_root}/vertex_subset_{vertex_subset_size}.npz')
191
+ vertex_subset = vertex_subset_dict['i_verts']
192
+ faces = vertex_subset_dict['faces']
193
+ joint_regressor_post_lbs = np.load(
194
+ f'{model_root}/vertex_subset_joint_regr_post_lbs_{vertex_subset_size}.npy'
195
+ )
196
+
197
+ if vertex_subset is None:
198
+ vertex_subset = np.arange(res['num_vertices'], dtype=np.int64)
199
+ else:
200
+ vertex_subset = np.array(vertex_subset, dtype=np.int64)
201
+
202
+ if faces is None:
203
+ faces = res['faces']
204
+
205
+ if joint_regressor_post_lbs is None:
206
+ joint_regressor_post_lbs = res['J_regressor']
207
+
208
+ return ModelData(
209
+ v_template=res['v_template'][vertex_subset],
210
+ shapedirs=res['shapedirs'][vertex_subset, :, :num_betas],
211
+ posedirs=res['posedirs'][vertex_subset],
212
+ J_regressor_post_lbs=joint_regressor_post_lbs,
213
+ J_template=res['J_template'],
214
+ J_shapedirs=res['J_shapedirs'][:, :, :num_betas],
215
+ kid_shapedir=res['kid_shapedir'][vertex_subset],
216
+ kid_J_shapedir=res['kid_J_shapedir'],
217
+ weights=res['weights'][vertex_subset],
218
+ kintree_parents=res['kintree_parents'],
219
+ faces=faces,
220
+ num_joints=res['num_joints'],
221
+ num_vertices=len(vertex_subset),
222
+ vertex_subset=vertex_subset,
223
+ )
224
+
225
+
226
+ @contextlib.contextmanager
227
+ def monkey_patched_for_chumpy():
228
+ """The pickle file of SMPLH imports chumpy and it tries to import np.bool etc which are
229
+ not available anymore.
230
+ """
231
+ added = []
232
+ for name in ['bool', 'int', 'object', 'str']:
233
+ if name not in dir(np):
234
+ try:
235
+ sys.modules[f'numpy.{name}'] = getattr(np, name + '_')
236
+ added.append(name)
237
+ except AttributeError:
238
+ pass
239
+
240
+ sys.modules['numpy.float'] = float
241
+ sys.modules['numpy.complex'] = np.complex128
242
+ sys.modules['numpy.NINF'] = -np.inf
243
+ np.NINF = -np.inf # type: ignore[misc]
244
+ np.complex = np.complex128 # type: ignore[misc]
245
+ np.float = float # type: ignore[misc]
246
+
247
+ if 'unicode' not in dir(np):
248
+ sys.modules['numpy.unicode'] = np.str_
249
+ added.append('unicode')
250
+
251
+ import inspect
252
+
253
+ added_getargspec = False
254
+ if not hasattr(inspect, 'getargspec'):
255
+ inspect.getargspec = inspect.getfullargspec
256
+ added_getargspec = True
257
+
258
+ with warnings.catch_warnings():
259
+ warnings.simplefilter('ignore', FutureWarning)
260
+ yield
261
+
262
+ for name in added:
263
+ del sys.modules[f'numpy.{name}']
264
+
265
+ if added_getargspec:
266
+ del inspect.getargspec
@@ -0,0 +1 @@
1
+ from __future__ import annotations
@@ -0,0 +1,55 @@
1
+ from __future__ import annotations
2
+ import os
3
+
4
+ import numpy as np
5
+ from .. import np as smplfitter_np
6
+ import trimesh
7
+ from scipy.optimize import linear_sum_assignment
8
+ from scipy.spatial.distance import cdist
9
+
10
+
11
+ def main():
12
+ DATA_ROOT = os.getenv('DATA_ROOT', default='.')
13
+ for model_name in ['smpl', 'smplx']:
14
+ model_root = f'{DATA_ROOT}/body_models/{model_name}'
15
+ bm = smplfitter_np.get_cached_body_model(model_name)
16
+ verts = bm.single()['vertices']
17
+
18
+ for n in [32, 64, 128, 256, 512, 1024]:
19
+ i_verts, faces = decimate(verts, bm.faces, n)
20
+ if len(i_verts) != n:
21
+ print(f'Failed to decimate to {n} vertices')
22
+ continue
23
+ else:
24
+ print(f'Decimated to {n} vertices')
25
+ np.savez(f'{model_root}/vertex_subset_{n}.npz', i_verts=i_verts, faces=faces)
26
+
27
+
28
+ def _decimate(verts, faces, n_verts_out=128):
29
+ n_faces = 2 * n_verts_out - 4
30
+ decimated_mesh = trimesh.Trimesh(verts, faces).simplify_quadric_decimation(face_count=n_faces)
31
+ row_ind, col_ind = linear_sum_assignment(cdist(verts, decimated_mesh.vertices))
32
+ i_verts = row_ind[np.argsort(col_ind)]
33
+ return i_verts, decimated_mesh.faces
34
+
35
+
36
+ def decimate(verts, faces, n_verts_out=128, n_trials=100):
37
+ # We may have to try it multiple times to get the desired number of vertices
38
+ # since the decimation algorithm may not always return the exact number of vertices we ask.
39
+ # The documentation explicitly states this.
40
+
41
+ n_verts_out_arg = n_verts_out
42
+ i_verts, new_faces = _decimate(verts, faces, n_verts_out_arg)
43
+
44
+ for _ in range(n_trials):
45
+ if i_verts.shape[0] == n_verts_out:
46
+ break
47
+
48
+ n_verts_out_arg += n_verts_out - i_verts.shape[0]
49
+ i_verts, new_faces = _decimate(verts, faces, n_verts_out_arg)
50
+
51
+ return i_verts, new_faces
52
+
53
+
54
+ if __name__ == '__main__':
55
+ main()
@@ -0,0 +1,290 @@
1
+ from __future__ import annotations
2
+ # This module finds joint regressors for subsets of body model vertices.
3
+ # That is, it finds a linear regressor that maps a subset of vertices to the joints.
4
+ # And this is done for the post-LBS case, i.e. the vertices and joints are posed with linear
5
+ # blend skinning.
6
+ # This is useful in body model fitting in case the user does not provide the joint positions.
7
+ # In that case we must run a regressor on the (already posed) vertices.
8
+ #
9
+ # The joint regressors need to create convex combinations, i.e. the weights must be non-negative
10
+ # and sum to 1 for each joint. They should also be sparse and spatially compact.
11
+ # That is, each joint should only depend on a few vertices that are spatially close to each other.
12
+ # This is achieved by adding a regularization term that encourages the weights to be
13
+ # sparse (L1/2 norm), and a term that computes the weighted variance of the template vertices.
14
+
15
+ import os
16
+
17
+ import numpy as np
18
+ from .. import pt as smplfitter_pt
19
+ import torch
20
+ import torch.nn as nn
21
+ import torch.optim as optim
22
+ from torch.utils.data import DataLoader, IterableDataset
23
+ from tqdm import tqdm
24
+
25
+ DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
26
+ DATA_ROOT = os.getenv('DATA_ROOT', default='.')
27
+ torch.backends.cudnn.benchmark = True
28
+
29
+
30
+ def main():
31
+ for body_model_name in ['smpl', 'smplx']:
32
+ canonical_verts = torch.from_numpy(
33
+ np.load(f'{DATA_ROOT}/nlf/canonical_vertices_{body_model_name}.npy')
34
+ ).float()
35
+ body_model = smplfitter_pt.BodyModel(body_model_name).to(DEVICE)
36
+ dataset = RandomBodyParamDataset(num_betas=16, num_joints=body_model.num_joints)
37
+ dataloader = DataLoader(dataset, batch_size=None, num_workers=4, pin_memory=True)
38
+
39
+ for n_verts_subset in reversed(
40
+ [32, 64, 128, 256, 512, 1024, 2048, 4096, body_model.num_vertices]
41
+ ):
42
+ print(f'Fitting joint regressor for {n_verts_subset} vertices')
43
+ out_path = (
44
+ f'{DATA_ROOT}/body_models/{body_model_name}/'
45
+ f'vertex_subset_joint_regr_post_lbs_{n_verts_subset}.npy'
46
+ )
47
+ # if os.path.exists(out_path):
48
+ # continue
49
+
50
+ if n_verts_subset == body_model.num_vertices:
51
+ i_verts = torch.arange(body_model.num_vertices)
52
+ else:
53
+ subset = np.load(
54
+ f'{DATA_ROOT}/body_models/{body_model_name}/vertex_subset_{n_verts_subset}.npz'
55
+ )
56
+ i_verts = torch.from_numpy(subset['i_verts'])
57
+
58
+ model = ConvexCombiningRegressor(len(i_verts), body_model.num_joints).to(DEVICE)
59
+ trainer = ConvexCombiningRegressorTrainer(
60
+ model=model,
61
+ body_model=body_model,
62
+ template_verts=canonical_verts[i_verts],
63
+ vertex_subset=i_verts,
64
+ regul_lambda=6e-5,
65
+ )
66
+ optimizer = optim.Adam(model.parameters(), lr=1e0)
67
+ scheduler = optim.lr_scheduler.LambdaLR(
68
+ optimizer, lr_lambda=lambda step: 1.0 if step < int(37500 * 0.9) else 1e-3
69
+ )
70
+ trainer.train_loop(
71
+ dataloader, total_steps=37500, optimizer=optimizer, scheduler=scheduler
72
+ )
73
+ model.threshold_for_sparsity(1e-3)
74
+
75
+ scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda step: 30)
76
+ trainer.train_loop(
77
+ dataloader, total_steps=37500 + 12500, optimizer=optimizer, scheduler=scheduler
78
+ )
79
+ model.threshold_for_sparsity(1e-3)
80
+
81
+ J_subset = model.get_w().cpu().detach().numpy().T
82
+ print(f'Sparsity ratio: {sparsity_ratio(J_subset)}')
83
+ np.save(out_path, J_subset)
84
+
85
+
86
+ class ConvexCombiningRegressor(nn.Module):
87
+ def __init__(self, n_in_points, n_out_points):
88
+ super().__init__()
89
+ self.n_in = n_in_points
90
+ self.n_out = n_out_points
91
+
92
+ self.weight_mask = nn.Parameter(torch.ones(n_in_points, n_out_points), requires_grad=False)
93
+ self.weights = nn.Parameter(torch.empty(n_in_points, n_out_points))
94
+ nn.init.uniform_(self.weights, -1, 1)
95
+
96
+ def forward(self, x):
97
+ w = self.get_w()
98
+ return torch.einsum('bjc,jJ->bJc', x, w)
99
+
100
+ def get_w(self):
101
+ w = torch.nn.functional.softplus(self.weights) * self.weight_mask
102
+ return w / w.sum(dim=0, keepdim=True)
103
+
104
+ def threshold_for_sparsity(self, threshold=1e-3):
105
+ with torch.no_grad():
106
+ self.weight_mask.data = (torch.abs(self.get_w()) > threshold).float()
107
+
108
+
109
+ class ConvexCombiningRegressorTrainer:
110
+ def __init__(self, model, body_model, template_verts, vertex_subset, regul_lambda=3e-5):
111
+ self.model = model
112
+ self.body_model = body_model
113
+ self.template_verts = template_verts.to(DEVICE)
114
+ self.vertex_subset = vertex_subset
115
+ self.regul_lambda = regul_lambda
116
+ self.current_step = 0
117
+
118
+ def train_loop(self, dataloader, total_steps, optimizer, scheduler=None):
119
+ progress_bar = tqdm(
120
+ total=total_steps,
121
+ desc='Training',
122
+ unit='step',
123
+ dynamic_ncols=True,
124
+ initial=self.current_step,
125
+ )
126
+
127
+ for batch in dataloader:
128
+ batch = EasyDict({k: v.to(DEVICE) for k, v in batch.items()})
129
+ need_metrics = self.current_step % 100 == 0
130
+ losses, metrics = self.train_step(
131
+ batch, optimizer, scheduler, need_metrics=need_metrics
132
+ )
133
+ self.current_step += 1
134
+ progress_bar.update(1)
135
+
136
+ if need_metrics:
137
+ losses_and_metrics_str = ', '.join(
138
+ f'{k}: {v.item():.4f}' for k, v in (losses | metrics).items()
139
+ )
140
+ progress_bar.set_postfix_str(losses_and_metrics_str)
141
+
142
+ if self.current_step >= total_steps:
143
+ break
144
+
145
+ def forward_train(self, inps):
146
+ preds = EasyDict()
147
+ r = self.body_model(inps.pose, inps.shape)
148
+ preds.postjoints = self.body_model.J_regressor_post_lbs @ r['vertices']
149
+ preds.pose3d = self.model(r['vertices'][:, self.vertex_subset])
150
+ preds.pose3d_gt = r['joints']
151
+ return preds
152
+
153
+ def train_step(self, inps, optimizer, scheduler=None, need_metrics=True):
154
+ self.model.train()
155
+ optimizer.zero_grad()
156
+ preds = self.forward_train(inps)
157
+ losses = self.compute_losses(inps, preds)
158
+ losses.loss.backward()
159
+ optimizer.step()
160
+ if scheduler is not None:
161
+ scheduler.step()
162
+
163
+ if need_metrics:
164
+ with torch.no_grad():
165
+ metrics = self.compute_metrics(inps, preds)
166
+ else:
167
+ metrics = None
168
+
169
+ return losses, metrics
170
+
171
+ def compute_losses(self, inps, preds):
172
+ losses = EasyDict()
173
+ losses.main_loss = torch.mean(torch.abs(preds.pose3d_gt - preds.pose3d))
174
+
175
+ w = self.model.get_w()
176
+ losses.regul = torch.sum(soft_sqrt(torch.abs(w), 1e-5)) / w.shape[1]
177
+ # losses.supp = mean_spatial_support(self.template_verts, w)
178
+ losses.loss = (
179
+ losses.main_loss + self.regul_lambda * losses.regul # + self.supp_lambda * losses.supp
180
+ )
181
+ return losses
182
+
183
+ def compute_metrics(self, inps, preds):
184
+ m = EasyDict()
185
+ dist = torch.norm(preds.pose3d_gt - preds.pose3d, dim=-1)
186
+ m.pck1 = pck(dist, 0.01)
187
+ m.pck2 = pck(dist, 0.02)
188
+ m.pck3 = pck(dist, 0.03)
189
+ m.pck7 = pck(dist, 0.07)
190
+ m.euclidean = torch.mean(dist)
191
+ m.l1 = torch.mean(torch.abs(preds.pose3d_gt - preds.pose3d))
192
+ m.max_supp = torch.max(
193
+ torch.sqrt(spatial_support(self.template_verts, self.model.get_w()))
194
+ )
195
+
196
+ dist = torch.norm(preds.pose3d_gt - preds.postjoints, dim=-1)
197
+ m.pck1_post = pck(dist, 0.01)
198
+ return m
199
+
200
+
201
+ class RandomBodyParamDataset(IterableDataset):
202
+ def __init__(self, batch_size=72, num_joints=24, num_betas=16):
203
+ self.batch_size = batch_size
204
+ self.num_joints = num_joints
205
+ self.num_betas = num_betas
206
+ self.rng = torch.Generator()
207
+
208
+ def __iter__(self):
209
+ with torch.no_grad():
210
+ while True:
211
+ pose = self.random_smpl_pose() / 3
212
+ shape = torch.empty((self.batch_size, self.num_betas))
213
+ nn.init.trunc_normal_(shape, mean=0.0, std=1.0, a=-2.0, b=2.0, generator=self.rng)
214
+ yield {'pose': pose, 'shape': shape * 10}
215
+
216
+ def random_rotvec(self, batch_size):
217
+ rand = torch.rand(3, batch_size, generator=self.rng)
218
+ r1 = torch.sqrt(1 - rand[0])
219
+ r2 = torch.sqrt(rand[0])
220
+ t1 = 2 * np.pi * rand[1]
221
+ t2 = 2 * np.pi * rand[2]
222
+ cost2 = torch.cos(t2)
223
+ xyz = torch.stack([torch.sin(t1) * r1, torch.cos(t1) * r1, torch.sin(t2) * r2], dim=0)
224
+ return (xyz / torch.sqrt(1 - cost2**2 * rand[0]) * 2 * torch.acos(cost2 * r2)).T
225
+
226
+ def random_smpl_pose(self):
227
+ return self.random_rotvec(self.batch_size * self.num_joints).reshape(
228
+ self.batch_size, self.num_joints * 3
229
+ )
230
+
231
+
232
+ def sparsity_ratio(J):
233
+ return np.count_nonzero(np.abs(J) > 1e-4) / J.shape[0]
234
+
235
+
236
+ def soft_sqrt(x, eps):
237
+ return x / torch.sqrt(x + eps)
238
+
239
+
240
+ def spatial_support(template, weights):
241
+ weighted_mean = torch.matmul(weights.t(), template)
242
+ sq_diff = (template[np.newaxis, :] - weighted_mean[:, np.newaxis]) ** 2
243
+ sq_dists = torch.sum(sq_diff, dim=-1)
244
+ return torch.einsum('Jj,jJ->J', sq_dists, torch.abs(weights))
245
+
246
+
247
+ def pck(x, t):
248
+ return (x <= t).float().mean()
249
+
250
+
251
+ class EasyDict(dict):
252
+ def __init__(self, d=None, **kwargs):
253
+ super().__init__()
254
+ if d is None:
255
+ d = {}
256
+ else:
257
+ d = dict(d)
258
+ if kwargs:
259
+ d.update(**kwargs)
260
+ for k, v in d.items():
261
+ setattr(self, k, v)
262
+ # Class attributes
263
+ for k in self.__class__.__dict__.keys():
264
+ if not (k.startswith('__') and k.endswith('__')) and k not in ('update', 'pop'):
265
+ setattr(self, k, getattr(self, k))
266
+
267
+ def __setattr__(self, name, value):
268
+ super().__setitem__(name, value)
269
+
270
+ def __getattr__(self, name):
271
+ try:
272
+ return super().__getitem__(name)
273
+ except KeyError:
274
+ raise AttributeError(name)
275
+
276
+ __setitem__ = __setattr__
277
+
278
+ def update(self, e=None, **f):
279
+ d = e or dict()
280
+ d.update(f)
281
+ for k, v in d.items():
282
+ setattr(self, k, v)
283
+
284
+ def pop(self, k, d=None):
285
+ delattr(self, k)
286
+ return super().pop(k, d)
287
+
288
+
289
+ if __name__ == '__main__':
290
+ main()