solarian-diffusion 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffusion_policy/__init__.py +0 -0
- diffusion_policy/codecs/__init__.py +0 -0
- diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
- diffusion_policy/common/__init__.py +0 -0
- diffusion_policy/common/checkpoint_util.py +64 -0
- diffusion_policy/common/json_logger.py +117 -0
- diffusion_policy/common/normalize_util.py +251 -0
- diffusion_policy/common/pose_repr_util.py +112 -0
- diffusion_policy/common/pytorch_util.py +82 -0
- diffusion_policy/common/q_critic_util.py +135 -0
- diffusion_policy/common/replay_buffer.py +672 -0
- diffusion_policy/common/sampler.py +204 -0
- diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
- diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
- diffusion_policy/config/legacy/task/can_image.yaml +64 -0
- diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
- diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
- diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
- diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
- diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
- diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
- diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
- diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
- diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
- diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
- diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
- diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
- diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
- diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
- diffusion_policy/config/legacy/task/square_image.yaml +64 -0
- diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
- diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
- diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
- diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
- diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
- diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
- diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
- diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
- diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
- diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
- diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
- diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
- diffusion_policy/config/legacy/task/umi.yaml +45 -0
- diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
- diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
- diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
- diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
- diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
- diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
- diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
- diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
- diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
- diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
- diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
- diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
- diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
- diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
- diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
- diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
- diffusion_policy/config/rl_annotate.yaml +265 -0
- diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
- diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
- diffusion_policy/config/rl_annotate_v2.yaml +141 -0
- diffusion_policy/config/rl_annotate_v3.yaml +165 -0
- diffusion_policy/config/task/lift.yaml +89 -0
- diffusion_policy/config/task/square.yaml +92 -0
- diffusion_policy/config/task/tool_hang.yaml +93 -0
- diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
- diffusion_policy/config/task/umi.yaml +105 -0
- diffusion_policy/config/task/umi_bimanual.yaml +135 -0
- diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
- diffusion_policy/config/task/umi_image.yaml +36 -0
- diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
- diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
- diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
- diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
- diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
- diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
- diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
- diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
- diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
- diffusion_policy/config/train_iql_umi.yaml +186 -0
- diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
- diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
- diffusion_policy/dataset/__init__.py +0 -0
- diffusion_policy/dataset/base_dataset.py +76 -0
- diffusion_policy/dataset/umi_dataset.py +652 -0
- diffusion_policy/env_runner/__init__.py +0 -0
- diffusion_policy/env_runner/base_image_runner.py +9 -0
- diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
- diffusion_policy/model/__init__.py +0 -0
- diffusion_policy/model/common/__init__.py +0 -0
- diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
- diffusion_policy/model/common/lr_scheduler.py +46 -0
- diffusion_policy/model/common/module_attr_mixin.py +14 -0
- diffusion_policy/model/common/normalizer.py +353 -0
- diffusion_policy/model/common/tensor_util.py +960 -0
- diffusion_policy/model/diffusion/__init__.py +0 -0
- diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
- diffusion_policy/model/diffusion/conv1d_components.py +46 -0
- diffusion_policy/model/diffusion/ema_model.py +88 -0
- diffusion_policy/model/diffusion/mask_generator.py +221 -0
- diffusion_policy/model/diffusion/positional_embedding.py +17 -0
- diffusion_policy/model/vision/__init__.py +0 -0
- diffusion_policy/model/vision/crop_randomizer.py +288 -0
- diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
- diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
- diffusion_policy/policy/__init__.py +0 -0
- diffusion_policy/policy/base_image_policy.py +27 -0
- diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
- diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
- diffusion_policy/policy/parl_action_optimizer.py +675 -0
- diffusion_policy/workspace/__init__.py +0 -0
- diffusion_policy/workspace/base_workspace.py +146 -0
- diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
- solarian_diffusion/__init__.py +32 -0
- solarian_diffusion/config.py +31 -0
- solarian_diffusion/policy_server.py +316 -0
- solarian_diffusion/rtc.py +272 -0
- solarian_diffusion/service.py +16 -0
- solarian_diffusion/trainer.py +213 -0
- solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
- solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
- solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
- solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
- solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
- solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
- solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
- umi/__init__.py +0 -0
- umi/common/__init__.py +0 -0
- umi/common/pose_util.py +115 -0
|
File without changes
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
from typing import Union
|
|
2
|
+
import logging
|
|
3
|
+
import torch
|
|
4
|
+
import torch.nn as nn
|
|
5
|
+
import einops
|
|
6
|
+
from einops.layers.torch import Rearrange
|
|
7
|
+
|
|
8
|
+
from diffusion_policy.model.diffusion.conv1d_components import (
|
|
9
|
+
Downsample1d, Upsample1d, Conv1dBlock)
|
|
10
|
+
from diffusion_policy.model.diffusion.positional_embedding import SinusoidalPosEmb
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
class ConditionalResidualBlock1D(nn.Module):
|
|
15
|
+
def __init__(self,
|
|
16
|
+
in_channels,
|
|
17
|
+
out_channels,
|
|
18
|
+
cond_dim,
|
|
19
|
+
kernel_size=3,
|
|
20
|
+
n_groups=8,
|
|
21
|
+
cond_predict_scale=False):
|
|
22
|
+
super().__init__()
|
|
23
|
+
|
|
24
|
+
self.blocks = nn.ModuleList([
|
|
25
|
+
Conv1dBlock(in_channels, out_channels, kernel_size, n_groups=n_groups),
|
|
26
|
+
Conv1dBlock(out_channels, out_channels, kernel_size, n_groups=n_groups),
|
|
27
|
+
])
|
|
28
|
+
|
|
29
|
+
# FiLM modulation https://arxiv.org/abs/1709.07871
|
|
30
|
+
# predicts per-channel scale and bias
|
|
31
|
+
cond_channels = out_channels
|
|
32
|
+
if cond_predict_scale:
|
|
33
|
+
cond_channels = out_channels * 2
|
|
34
|
+
self.cond_predict_scale = cond_predict_scale
|
|
35
|
+
self.out_channels = out_channels
|
|
36
|
+
self.cond_encoder = nn.Sequential(
|
|
37
|
+
nn.Mish(),
|
|
38
|
+
nn.Linear(cond_dim, cond_channels),
|
|
39
|
+
Rearrange('batch t -> batch t 1'),
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
# make sure dimensions compatible
|
|
43
|
+
self.residual_conv = nn.Conv1d(in_channels, out_channels, 1) \
|
|
44
|
+
if in_channels != out_channels else nn.Identity()
|
|
45
|
+
|
|
46
|
+
def forward(self, x, cond):
|
|
47
|
+
'''
|
|
48
|
+
x : [ batch_size x in_channels x horizon ]
|
|
49
|
+
cond : [ batch_size x cond_dim]
|
|
50
|
+
|
|
51
|
+
returns:
|
|
52
|
+
out : [ batch_size x out_channels x horizon ]
|
|
53
|
+
'''
|
|
54
|
+
out = self.blocks[0](x)
|
|
55
|
+
embed = self.cond_encoder(cond)
|
|
56
|
+
if self.cond_predict_scale:
|
|
57
|
+
embed = embed.reshape(
|
|
58
|
+
embed.shape[0], 2, self.out_channels, 1)
|
|
59
|
+
scale = embed[:,0,...]
|
|
60
|
+
bias = embed[:,1,...]
|
|
61
|
+
out = scale * out + bias
|
|
62
|
+
else:
|
|
63
|
+
out = out + embed
|
|
64
|
+
out = self.blocks[1](out)
|
|
65
|
+
out = out + self.residual_conv(x)
|
|
66
|
+
return out
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class ConditionalUnet1D(nn.Module):
|
|
70
|
+
def __init__(self,
|
|
71
|
+
input_dim,
|
|
72
|
+
local_cond_dim=None,
|
|
73
|
+
global_cond_dim=None,
|
|
74
|
+
diffusion_step_embed_dim=256,
|
|
75
|
+
down_dims=[256,512,1024],
|
|
76
|
+
kernel_size=3,
|
|
77
|
+
n_groups=8,
|
|
78
|
+
cond_predict_scale=False
|
|
79
|
+
):
|
|
80
|
+
super().__init__()
|
|
81
|
+
all_dims = [input_dim] + list(down_dims)
|
|
82
|
+
start_dim = down_dims[0]
|
|
83
|
+
|
|
84
|
+
dsed = diffusion_step_embed_dim
|
|
85
|
+
diffusion_step_encoder = nn.Sequential(
|
|
86
|
+
SinusoidalPosEmb(dsed),
|
|
87
|
+
nn.Linear(dsed, dsed * 4),
|
|
88
|
+
nn.Mish(),
|
|
89
|
+
nn.Linear(dsed * 4, dsed),
|
|
90
|
+
)
|
|
91
|
+
cond_dim = dsed
|
|
92
|
+
if global_cond_dim is not None:
|
|
93
|
+
cond_dim += global_cond_dim
|
|
94
|
+
|
|
95
|
+
in_out = list(zip(all_dims[:-1], all_dims[1:]))
|
|
96
|
+
|
|
97
|
+
local_cond_encoder = None
|
|
98
|
+
if local_cond_dim is not None:
|
|
99
|
+
_, dim_out = in_out[0]
|
|
100
|
+
dim_in = local_cond_dim
|
|
101
|
+
local_cond_encoder = nn.ModuleList([
|
|
102
|
+
# down encoder
|
|
103
|
+
ConditionalResidualBlock1D(
|
|
104
|
+
dim_in, dim_out, cond_dim=cond_dim,
|
|
105
|
+
kernel_size=kernel_size, n_groups=n_groups,
|
|
106
|
+
cond_predict_scale=cond_predict_scale),
|
|
107
|
+
# up encoder
|
|
108
|
+
ConditionalResidualBlock1D(
|
|
109
|
+
dim_in, dim_out, cond_dim=cond_dim,
|
|
110
|
+
kernel_size=kernel_size, n_groups=n_groups,
|
|
111
|
+
cond_predict_scale=cond_predict_scale)
|
|
112
|
+
])
|
|
113
|
+
|
|
114
|
+
mid_dim = all_dims[-1]
|
|
115
|
+
self.mid_modules = nn.ModuleList([
|
|
116
|
+
ConditionalResidualBlock1D(
|
|
117
|
+
mid_dim, mid_dim, cond_dim=cond_dim,
|
|
118
|
+
kernel_size=kernel_size, n_groups=n_groups,
|
|
119
|
+
cond_predict_scale=cond_predict_scale
|
|
120
|
+
),
|
|
121
|
+
ConditionalResidualBlock1D(
|
|
122
|
+
mid_dim, mid_dim, cond_dim=cond_dim,
|
|
123
|
+
kernel_size=kernel_size, n_groups=n_groups,
|
|
124
|
+
cond_predict_scale=cond_predict_scale
|
|
125
|
+
),
|
|
126
|
+
])
|
|
127
|
+
|
|
128
|
+
down_modules = nn.ModuleList([])
|
|
129
|
+
for ind, (dim_in, dim_out) in enumerate(in_out):
|
|
130
|
+
is_last = ind >= (len(in_out) - 1)
|
|
131
|
+
down_modules.append(nn.ModuleList([
|
|
132
|
+
ConditionalResidualBlock1D(
|
|
133
|
+
dim_in, dim_out, cond_dim=cond_dim,
|
|
134
|
+
kernel_size=kernel_size, n_groups=n_groups,
|
|
135
|
+
cond_predict_scale=cond_predict_scale),
|
|
136
|
+
ConditionalResidualBlock1D(
|
|
137
|
+
dim_out, dim_out, cond_dim=cond_dim,
|
|
138
|
+
kernel_size=kernel_size, n_groups=n_groups,
|
|
139
|
+
cond_predict_scale=cond_predict_scale),
|
|
140
|
+
Downsample1d(dim_out) if not is_last else nn.Identity()
|
|
141
|
+
]))
|
|
142
|
+
|
|
143
|
+
up_modules = nn.ModuleList([])
|
|
144
|
+
for ind, (dim_in, dim_out) in enumerate(reversed(in_out[1:])):
|
|
145
|
+
is_last = ind >= (len(in_out) - 1)
|
|
146
|
+
up_modules.append(nn.ModuleList([
|
|
147
|
+
ConditionalResidualBlock1D(
|
|
148
|
+
dim_out*2, dim_in, cond_dim=cond_dim,
|
|
149
|
+
kernel_size=kernel_size, n_groups=n_groups,
|
|
150
|
+
cond_predict_scale=cond_predict_scale),
|
|
151
|
+
ConditionalResidualBlock1D(
|
|
152
|
+
dim_in, dim_in, cond_dim=cond_dim,
|
|
153
|
+
kernel_size=kernel_size, n_groups=n_groups,
|
|
154
|
+
cond_predict_scale=cond_predict_scale),
|
|
155
|
+
Upsample1d(dim_in) if not is_last else nn.Identity()
|
|
156
|
+
]))
|
|
157
|
+
|
|
158
|
+
final_conv = nn.Sequential(
|
|
159
|
+
Conv1dBlock(start_dim, start_dim, kernel_size=kernel_size),
|
|
160
|
+
nn.Conv1d(start_dim, input_dim, 1),
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
self.diffusion_step_encoder = diffusion_step_encoder
|
|
164
|
+
self.local_cond_encoder = local_cond_encoder
|
|
165
|
+
self.up_modules = up_modules
|
|
166
|
+
self.down_modules = down_modules
|
|
167
|
+
self.final_conv = final_conv
|
|
168
|
+
|
|
169
|
+
logger.info(
|
|
170
|
+
"number of parameters: %e", sum(p.numel() for p in self.parameters())
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
def forward(self,
|
|
174
|
+
sample: torch.Tensor,
|
|
175
|
+
timestep: Union[torch.Tensor, float, int],
|
|
176
|
+
local_cond=None, global_cond=None, **kwargs):
|
|
177
|
+
"""
|
|
178
|
+
x: (B,T,input_dim)
|
|
179
|
+
timestep: (B,) or int, diffusion step
|
|
180
|
+
local_cond: (B,T,local_cond_dim)
|
|
181
|
+
global_cond: (B,global_cond_dim)
|
|
182
|
+
output: (B,T,input_dim)
|
|
183
|
+
"""
|
|
184
|
+
sample = einops.rearrange(sample, 'b h t -> b t h')
|
|
185
|
+
|
|
186
|
+
# 1. time
|
|
187
|
+
timesteps = timestep
|
|
188
|
+
if not torch.is_tensor(timesteps):
|
|
189
|
+
# TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can
|
|
190
|
+
timesteps = torch.tensor([timesteps], dtype=torch.long, device=sample.device)
|
|
191
|
+
elif torch.is_tensor(timesteps) and len(timesteps.shape) == 0:
|
|
192
|
+
timesteps = timesteps[None].to(sample.device)
|
|
193
|
+
# broadcast to batch dimension in a way that's compatible with ONNX/Core ML
|
|
194
|
+
timesteps = timesteps.expand(sample.shape[0])
|
|
195
|
+
|
|
196
|
+
global_feature = self.diffusion_step_encoder(timesteps)
|
|
197
|
+
|
|
198
|
+
if global_cond is not None:
|
|
199
|
+
global_feature = torch.cat([
|
|
200
|
+
global_feature, global_cond
|
|
201
|
+
], axis=-1)
|
|
202
|
+
|
|
203
|
+
# encode local features
|
|
204
|
+
h_local = list()
|
|
205
|
+
if local_cond is not None:
|
|
206
|
+
local_cond = einops.rearrange(local_cond, 'b h t -> b t h')
|
|
207
|
+
resnet, resnet2 = self.local_cond_encoder
|
|
208
|
+
x = resnet(local_cond, global_feature)
|
|
209
|
+
h_local.append(x)
|
|
210
|
+
x = resnet2(local_cond, global_feature)
|
|
211
|
+
h_local.append(x)
|
|
212
|
+
|
|
213
|
+
x = sample
|
|
214
|
+
h = []
|
|
215
|
+
for idx, (resnet, resnet2, downsample) in enumerate(self.down_modules):
|
|
216
|
+
x = resnet(x, global_feature)
|
|
217
|
+
if idx == 0 and len(h_local) > 0:
|
|
218
|
+
x = x + h_local[0]
|
|
219
|
+
x = resnet2(x, global_feature)
|
|
220
|
+
h.append(x)
|
|
221
|
+
x = downsample(x)
|
|
222
|
+
|
|
223
|
+
for mid_module in self.mid_modules:
|
|
224
|
+
x = mid_module(x, global_feature)
|
|
225
|
+
|
|
226
|
+
for idx, (resnet, resnet2, upsample) in enumerate(self.up_modules):
|
|
227
|
+
x = torch.cat((x, h.pop()), dim=1)
|
|
228
|
+
x = resnet(x, global_feature)
|
|
229
|
+
if idx == len(self.up_modules) and len(h_local) > 0:
|
|
230
|
+
x = x + h_local[1]
|
|
231
|
+
x = resnet2(x, global_feature)
|
|
232
|
+
x = upsample(x)
|
|
233
|
+
|
|
234
|
+
x = self.final_conv(x)
|
|
235
|
+
|
|
236
|
+
x = einops.rearrange(x, 'b t h -> b h t')
|
|
237
|
+
return x
|
|
238
|
+
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
import torch.nn as nn
|
|
3
|
+
import torch.nn.functional as F
|
|
4
|
+
# from einops.layers.torch import Rearrange
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Downsample1d(nn.Module):
|
|
8
|
+
def __init__(self, dim):
|
|
9
|
+
super().__init__()
|
|
10
|
+
self.conv = nn.Conv1d(dim, dim, 3, 2, 1)
|
|
11
|
+
|
|
12
|
+
def forward(self, x):
|
|
13
|
+
return self.conv(x)
|
|
14
|
+
|
|
15
|
+
class Upsample1d(nn.Module):
|
|
16
|
+
def __init__(self, dim):
|
|
17
|
+
super().__init__()
|
|
18
|
+
self.conv = nn.ConvTranspose1d(dim, dim, 4, 2, 1)
|
|
19
|
+
|
|
20
|
+
def forward(self, x):
|
|
21
|
+
return self.conv(x)
|
|
22
|
+
|
|
23
|
+
class Conv1dBlock(nn.Module):
|
|
24
|
+
'''
|
|
25
|
+
Conv1d --> GroupNorm --> Mish
|
|
26
|
+
'''
|
|
27
|
+
|
|
28
|
+
def __init__(self, inp_channels, out_channels, kernel_size, n_groups=8):
|
|
29
|
+
super().__init__()
|
|
30
|
+
|
|
31
|
+
self.block = nn.Sequential(
|
|
32
|
+
nn.Conv1d(inp_channels, out_channels, kernel_size, padding=kernel_size // 2),
|
|
33
|
+
# Rearrange('batch channels horizon -> batch channels 1 horizon'),
|
|
34
|
+
nn.GroupNorm(n_groups, out_channels),
|
|
35
|
+
# Rearrange('batch channels 1 horizon -> batch channels horizon'),
|
|
36
|
+
nn.Mish(),
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
def forward(self, x):
|
|
40
|
+
return self.block(x)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test():
|
|
44
|
+
cb = Conv1dBlock(256, 128, kernel_size=3)
|
|
45
|
+
x = torch.zeros((1,256,16))
|
|
46
|
+
o = cb(x)
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import copy
|
|
2
|
+
import torch
|
|
3
|
+
from torch.nn.modules.batchnorm import _BatchNorm
|
|
4
|
+
|
|
5
|
+
class EMAModel:
|
|
6
|
+
"""
|
|
7
|
+
Exponential Moving Average of models weights
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
def __init__(
|
|
11
|
+
self,
|
|
12
|
+
model,
|
|
13
|
+
update_after_step=0,
|
|
14
|
+
inv_gamma=1.0,
|
|
15
|
+
power=2 / 3,
|
|
16
|
+
min_value=0.0,
|
|
17
|
+
max_value=0.9999
|
|
18
|
+
):
|
|
19
|
+
"""
|
|
20
|
+
@crowsonkb's notes on EMA Warmup:
|
|
21
|
+
If gamma=1 and power=1, implements a simple average. gamma=1, power=2/3 are good values for models you plan
|
|
22
|
+
to train for a million or more steps (reaches decay factor 0.999 at 31.6K steps, 0.9999 at 1M steps),
|
|
23
|
+
gamma=1, power=3/4 for models you plan to train for less (reaches decay factor 0.999 at 10K steps, 0.9999
|
|
24
|
+
at 215.4k steps).
|
|
25
|
+
Args:
|
|
26
|
+
inv_gamma (float): Inverse multiplicative factor of EMA warmup. Default: 1.
|
|
27
|
+
power (float): Exponential factor of EMA warmup. Default: 2/3.
|
|
28
|
+
min_value (float): The minimum EMA decay rate. Default: 0.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
self.averaged_model = model
|
|
32
|
+
self.averaged_model.eval()
|
|
33
|
+
self.averaged_model.requires_grad_(False)
|
|
34
|
+
|
|
35
|
+
self.update_after_step = update_after_step
|
|
36
|
+
self.inv_gamma = inv_gamma
|
|
37
|
+
self.power = power
|
|
38
|
+
self.min_value = min_value
|
|
39
|
+
self.max_value = max_value
|
|
40
|
+
|
|
41
|
+
self.decay = 0.0
|
|
42
|
+
self.optimization_step = 0
|
|
43
|
+
|
|
44
|
+
def get_decay(self, optimization_step):
|
|
45
|
+
"""
|
|
46
|
+
Compute the decay factor for the exponential moving average.
|
|
47
|
+
"""
|
|
48
|
+
step = max(0, optimization_step - self.update_after_step - 1)
|
|
49
|
+
value = 1 - (1 + step / self.inv_gamma) ** -self.power
|
|
50
|
+
|
|
51
|
+
if step <= 0:
|
|
52
|
+
return 0.0
|
|
53
|
+
|
|
54
|
+
return max(self.min_value, min(value, self.max_value))
|
|
55
|
+
|
|
56
|
+
@torch.no_grad()
|
|
57
|
+
def step(self, new_model):
|
|
58
|
+
self.decay = self.get_decay(self.optimization_step)
|
|
59
|
+
|
|
60
|
+
# old_all_dataptrs = set()
|
|
61
|
+
# for param in new_model.parameters():
|
|
62
|
+
# data_ptr = param.data_ptr()
|
|
63
|
+
# if data_ptr != 0:
|
|
64
|
+
# old_all_dataptrs.add(data_ptr)
|
|
65
|
+
|
|
66
|
+
all_dataptrs = set()
|
|
67
|
+
for module, ema_module in zip(new_model.modules(), self.averaged_model.modules()):
|
|
68
|
+
for param, ema_param in zip(module.parameters(recurse=False), ema_module.parameters(recurse=False)):
|
|
69
|
+
# iterative over immediate parameters only.
|
|
70
|
+
if isinstance(param, dict):
|
|
71
|
+
raise RuntimeError('Dict parameter not supported')
|
|
72
|
+
|
|
73
|
+
# data_ptr = param.data_ptr()
|
|
74
|
+
# if data_ptr != 0:
|
|
75
|
+
# all_dataptrs.add(data_ptr)
|
|
76
|
+
|
|
77
|
+
if isinstance(module, _BatchNorm):
|
|
78
|
+
# skip batchnorms
|
|
79
|
+
ema_param.copy_(param.to(dtype=ema_param.dtype).data)
|
|
80
|
+
elif not param.requires_grad:
|
|
81
|
+
ema_param.copy_(param.to(dtype=ema_param.dtype).data)
|
|
82
|
+
else:
|
|
83
|
+
ema_param.mul_(self.decay)
|
|
84
|
+
ema_param.add_(param.data.to(dtype=ema_param.dtype), alpha=1 - self.decay)
|
|
85
|
+
|
|
86
|
+
# verify that iterating over module and then parameters is identical to parameters recursively.
|
|
87
|
+
# assert old_all_dataptrs == all_dataptrs
|
|
88
|
+
self.optimization_step += 1
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
from typing import Sequence, Optional
|
|
2
|
+
import torch
|
|
3
|
+
from torch import nn
|
|
4
|
+
from diffusion_policy.model.common.module_attr_mixin import ModuleAttrMixin
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def get_intersection_slice_mask(
|
|
8
|
+
shape: tuple,
|
|
9
|
+
dim_slices: Sequence[slice],
|
|
10
|
+
device: Optional[torch.device]=None
|
|
11
|
+
):
|
|
12
|
+
assert(len(shape) == len(dim_slices))
|
|
13
|
+
mask = torch.zeros(size=shape, dtype=torch.bool, device=device)
|
|
14
|
+
mask[dim_slices] = True
|
|
15
|
+
return mask
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def get_union_slice_mask(
|
|
19
|
+
shape: tuple,
|
|
20
|
+
dim_slices: Sequence[slice],
|
|
21
|
+
device: Optional[torch.device]=None
|
|
22
|
+
):
|
|
23
|
+
assert(len(shape) == len(dim_slices))
|
|
24
|
+
mask = torch.zeros(size=shape, dtype=torch.bool, device=device)
|
|
25
|
+
for i in range(len(dim_slices)):
|
|
26
|
+
this_slices = [slice(None)] * len(shape)
|
|
27
|
+
this_slices[i] = dim_slices[i]
|
|
28
|
+
mask[this_slices] = True
|
|
29
|
+
return mask
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class DummyMaskGenerator(ModuleAttrMixin):
|
|
33
|
+
def __init__(self):
|
|
34
|
+
super().__init__()
|
|
35
|
+
|
|
36
|
+
@torch.no_grad()
|
|
37
|
+
def forward(self, shape):
|
|
38
|
+
device = self.device
|
|
39
|
+
mask = torch.ones(size=shape, dtype=torch.bool, device=device)
|
|
40
|
+
return mask
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class LowdimMaskGenerator(ModuleAttrMixin):
|
|
44
|
+
def __init__(self,
|
|
45
|
+
action_dim, obs_dim,
|
|
46
|
+
# obs mask setup
|
|
47
|
+
max_n_obs_steps=2,
|
|
48
|
+
fix_obs_steps=True,
|
|
49
|
+
# action mask
|
|
50
|
+
action_visible=False
|
|
51
|
+
):
|
|
52
|
+
super().__init__()
|
|
53
|
+
self.action_dim = action_dim
|
|
54
|
+
self.obs_dim = obs_dim
|
|
55
|
+
self.max_n_obs_steps = max_n_obs_steps
|
|
56
|
+
self.fix_obs_steps = fix_obs_steps
|
|
57
|
+
self.action_visible = action_visible
|
|
58
|
+
|
|
59
|
+
@torch.no_grad()
|
|
60
|
+
def forward(self, shape, seed=None):
|
|
61
|
+
device = self.device
|
|
62
|
+
B, T, D = shape
|
|
63
|
+
assert D == (self.action_dim + self.obs_dim)
|
|
64
|
+
|
|
65
|
+
# create all tensors on this device
|
|
66
|
+
rng = torch.Generator(device=device)
|
|
67
|
+
if seed is not None:
|
|
68
|
+
rng = rng.manual_seed(seed)
|
|
69
|
+
|
|
70
|
+
# generate dim mask
|
|
71
|
+
dim_mask = torch.zeros(size=shape,
|
|
72
|
+
dtype=torch.bool, device=device)
|
|
73
|
+
is_action_dim = dim_mask.clone()
|
|
74
|
+
is_action_dim[...,:self.action_dim] = True
|
|
75
|
+
is_obs_dim = ~is_action_dim
|
|
76
|
+
|
|
77
|
+
# generate obs mask
|
|
78
|
+
if self.fix_obs_steps:
|
|
79
|
+
obs_steps = torch.full((B,),
|
|
80
|
+
fill_value=self.max_n_obs_steps, device=device)
|
|
81
|
+
else:
|
|
82
|
+
obs_steps = torch.randint(
|
|
83
|
+
low=1, high=self.max_n_obs_steps+1,
|
|
84
|
+
size=(B,), generator=rng, device=device)
|
|
85
|
+
|
|
86
|
+
steps = torch.arange(0, T, device=device).reshape(1,T).expand(B,T)
|
|
87
|
+
obs_mask = (steps.T < obs_steps).T.reshape(B,T,1).expand(B,T,D)
|
|
88
|
+
obs_mask = obs_mask & is_obs_dim
|
|
89
|
+
|
|
90
|
+
# generate action mask
|
|
91
|
+
if self.action_visible:
|
|
92
|
+
action_steps = torch.maximum(
|
|
93
|
+
obs_steps - 1,
|
|
94
|
+
torch.tensor(0,
|
|
95
|
+
dtype=obs_steps.dtype,
|
|
96
|
+
device=obs_steps.device))
|
|
97
|
+
action_mask = (steps.T < action_steps).T.reshape(B,T,1).expand(B,T,D)
|
|
98
|
+
action_mask = action_mask & is_action_dim
|
|
99
|
+
|
|
100
|
+
mask = obs_mask
|
|
101
|
+
if self.action_visible:
|
|
102
|
+
mask = mask | action_mask
|
|
103
|
+
|
|
104
|
+
return mask
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class KeypointMaskGenerator(ModuleAttrMixin):
|
|
108
|
+
def __init__(self,
|
|
109
|
+
# dimensions
|
|
110
|
+
action_dim, keypoint_dim,
|
|
111
|
+
# obs mask setup
|
|
112
|
+
max_n_obs_steps=2, fix_obs_steps=True,
|
|
113
|
+
# keypoint mask setup
|
|
114
|
+
keypoint_visible_rate=0.7, time_independent=False,
|
|
115
|
+
# action mask
|
|
116
|
+
action_visible=False,
|
|
117
|
+
context_dim=0, # dim for context
|
|
118
|
+
n_context_steps=1
|
|
119
|
+
):
|
|
120
|
+
super().__init__()
|
|
121
|
+
self.action_dim = action_dim
|
|
122
|
+
self.keypoint_dim = keypoint_dim
|
|
123
|
+
self.context_dim = context_dim
|
|
124
|
+
self.max_n_obs_steps = max_n_obs_steps
|
|
125
|
+
self.fix_obs_steps = fix_obs_steps
|
|
126
|
+
self.keypoint_visible_rate = keypoint_visible_rate
|
|
127
|
+
self.time_independent = time_independent
|
|
128
|
+
self.action_visible = action_visible
|
|
129
|
+
self.n_context_steps = n_context_steps
|
|
130
|
+
|
|
131
|
+
@torch.no_grad()
|
|
132
|
+
def forward(self, shape, seed=None):
|
|
133
|
+
device = self.device
|
|
134
|
+
B, T, D = shape
|
|
135
|
+
all_keypoint_dims = D - self.action_dim - self.context_dim
|
|
136
|
+
n_keypoints = all_keypoint_dims // self.keypoint_dim
|
|
137
|
+
|
|
138
|
+
# create all tensors on this device
|
|
139
|
+
rng = torch.Generator(device=device)
|
|
140
|
+
if seed is not None:
|
|
141
|
+
rng = rng.manual_seed(seed)
|
|
142
|
+
|
|
143
|
+
# generate dim mask
|
|
144
|
+
dim_mask = torch.zeros(size=shape,
|
|
145
|
+
dtype=torch.bool, device=device)
|
|
146
|
+
is_action_dim = dim_mask.clone()
|
|
147
|
+
is_action_dim[...,:self.action_dim] = True
|
|
148
|
+
is_context_dim = dim_mask.clone()
|
|
149
|
+
if self.context_dim > 0:
|
|
150
|
+
is_context_dim[...,-self.context_dim:] = True
|
|
151
|
+
is_obs_dim = ~(is_action_dim | is_context_dim)
|
|
152
|
+
# assumption trajectory=cat([action, keypoints, context], dim=-1)
|
|
153
|
+
|
|
154
|
+
# generate obs mask
|
|
155
|
+
if self.fix_obs_steps:
|
|
156
|
+
obs_steps = torch.full((B,),
|
|
157
|
+
fill_value=self.max_n_obs_steps, device=device)
|
|
158
|
+
else:
|
|
159
|
+
obs_steps = torch.randint(
|
|
160
|
+
low=1, high=self.max_n_obs_steps+1,
|
|
161
|
+
size=(B,), generator=rng, device=device)
|
|
162
|
+
|
|
163
|
+
steps = torch.arange(0, T, device=device).reshape(1,T).expand(B,T)
|
|
164
|
+
obs_mask = (steps.T < obs_steps).T.reshape(B,T,1).expand(B,T,D)
|
|
165
|
+
obs_mask = obs_mask & is_obs_dim
|
|
166
|
+
|
|
167
|
+
# generate action mask
|
|
168
|
+
if self.action_visible:
|
|
169
|
+
action_steps = torch.maximum(
|
|
170
|
+
obs_steps - 1,
|
|
171
|
+
torch.tensor(0,
|
|
172
|
+
dtype=obs_steps.dtype,
|
|
173
|
+
device=obs_steps.device))
|
|
174
|
+
action_mask = (steps.T < action_steps).T.reshape(B,T,1).expand(B,T,D)
|
|
175
|
+
action_mask = action_mask & is_action_dim
|
|
176
|
+
|
|
177
|
+
# generate keypoint mask
|
|
178
|
+
if self.time_independent:
|
|
179
|
+
visible_kps = torch.rand(size=(B, T, n_keypoints),
|
|
180
|
+
generator=rng, device=device) < self.keypoint_visible_rate
|
|
181
|
+
visible_dims = torch.repeat_interleave(visible_kps, repeats=self.keypoint_dim, dim=-1)
|
|
182
|
+
visible_dims_mask = torch.cat([
|
|
183
|
+
torch.ones((B, T, self.action_dim),
|
|
184
|
+
dtype=torch.bool, device=device),
|
|
185
|
+
visible_dims,
|
|
186
|
+
torch.ones((B, T, self.context_dim),
|
|
187
|
+
dtype=torch.bool, device=device),
|
|
188
|
+
], axis=-1)
|
|
189
|
+
keypoint_mask = visible_dims_mask
|
|
190
|
+
else:
|
|
191
|
+
visible_kps = torch.rand(size=(B,n_keypoints),
|
|
192
|
+
generator=rng, device=device) < self.keypoint_visible_rate
|
|
193
|
+
visible_dims = torch.repeat_interleave(visible_kps, repeats=self.keypoint_dim, dim=-1)
|
|
194
|
+
visible_dims_mask = torch.cat([
|
|
195
|
+
torch.ones((B, self.action_dim),
|
|
196
|
+
dtype=torch.bool, device=device),
|
|
197
|
+
visible_dims,
|
|
198
|
+
torch.ones((B, self.context_dim),
|
|
199
|
+
dtype=torch.bool, device=device),
|
|
200
|
+
], axis=-1)
|
|
201
|
+
keypoint_mask = visible_dims_mask.reshape(B,1,D).expand(B,T,D)
|
|
202
|
+
keypoint_mask = keypoint_mask & is_obs_dim
|
|
203
|
+
|
|
204
|
+
# generate context mask
|
|
205
|
+
context_mask = is_context_dim.clone()
|
|
206
|
+
context_mask[:,self.n_context_steps:,:] = False
|
|
207
|
+
|
|
208
|
+
mask = obs_mask & keypoint_mask
|
|
209
|
+
if self.action_visible:
|
|
210
|
+
mask = mask | action_mask
|
|
211
|
+
if self.context_dim > 0:
|
|
212
|
+
mask = mask | context_mask
|
|
213
|
+
|
|
214
|
+
return mask
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def test():
|
|
218
|
+
# kmg = KeypointMaskGenerator(2,2, random_obs_steps=True)
|
|
219
|
+
# self = KeypointMaskGenerator(2,2,context_dim=2, action_visible=True)
|
|
220
|
+
# self = KeypointMaskGenerator(2,2,context_dim=0, action_visible=True)
|
|
221
|
+
self = LowdimMaskGenerator(2,20, max_n_obs_steps=3, action_visible=True)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import math
|
|
2
|
+
import torch
|
|
3
|
+
import torch.nn as nn
|
|
4
|
+
|
|
5
|
+
class SinusoidalPosEmb(nn.Module):
|
|
6
|
+
def __init__(self, dim):
|
|
7
|
+
super().__init__()
|
|
8
|
+
self.dim = dim
|
|
9
|
+
|
|
10
|
+
def forward(self, x):
|
|
11
|
+
device = x.device
|
|
12
|
+
half_dim = self.dim // 2
|
|
13
|
+
emb = math.log(10000) / (half_dim - 1)
|
|
14
|
+
emb = torch.exp(torch.arange(half_dim, device=device) * -emb)
|
|
15
|
+
emb = x[:, None] * emb[None, :]
|
|
16
|
+
emb = torch.cat((emb.sin(), emb.cos()), dim=-1)
|
|
17
|
+
return emb
|
|
File without changes
|