solarian-diffusion 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffusion_policy/__init__.py +0 -0
- diffusion_policy/codecs/__init__.py +0 -0
- diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
- diffusion_policy/common/__init__.py +0 -0
- diffusion_policy/common/checkpoint_util.py +64 -0
- diffusion_policy/common/json_logger.py +117 -0
- diffusion_policy/common/normalize_util.py +251 -0
- diffusion_policy/common/pose_repr_util.py +112 -0
- diffusion_policy/common/pytorch_util.py +82 -0
- diffusion_policy/common/q_critic_util.py +135 -0
- diffusion_policy/common/replay_buffer.py +672 -0
- diffusion_policy/common/sampler.py +204 -0
- diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
- diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
- diffusion_policy/config/legacy/task/can_image.yaml +64 -0
- diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
- diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
- diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
- diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
- diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
- diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
- diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
- diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
- diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
- diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
- diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
- diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
- diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
- diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
- diffusion_policy/config/legacy/task/square_image.yaml +64 -0
- diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
- diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
- diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
- diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
- diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
- diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
- diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
- diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
- diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
- diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
- diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
- diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
- diffusion_policy/config/legacy/task/umi.yaml +45 -0
- diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
- diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
- diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
- diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
- diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
- diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
- diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
- diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
- diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
- diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
- diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
- diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
- diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
- diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
- diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
- diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
- diffusion_policy/config/rl_annotate.yaml +265 -0
- diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
- diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
- diffusion_policy/config/rl_annotate_v2.yaml +141 -0
- diffusion_policy/config/rl_annotate_v3.yaml +165 -0
- diffusion_policy/config/task/lift.yaml +89 -0
- diffusion_policy/config/task/square.yaml +92 -0
- diffusion_policy/config/task/tool_hang.yaml +93 -0
- diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
- diffusion_policy/config/task/umi.yaml +105 -0
- diffusion_policy/config/task/umi_bimanual.yaml +135 -0
- diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
- diffusion_policy/config/task/umi_image.yaml +36 -0
- diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
- diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
- diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
- diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
- diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
- diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
- diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
- diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
- diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
- diffusion_policy/config/train_iql_umi.yaml +186 -0
- diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
- diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
- diffusion_policy/dataset/__init__.py +0 -0
- diffusion_policy/dataset/base_dataset.py +76 -0
- diffusion_policy/dataset/umi_dataset.py +652 -0
- diffusion_policy/env_runner/__init__.py +0 -0
- diffusion_policy/env_runner/base_image_runner.py +9 -0
- diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
- diffusion_policy/model/__init__.py +0 -0
- diffusion_policy/model/common/__init__.py +0 -0
- diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
- diffusion_policy/model/common/lr_scheduler.py +46 -0
- diffusion_policy/model/common/module_attr_mixin.py +14 -0
- diffusion_policy/model/common/normalizer.py +353 -0
- diffusion_policy/model/common/tensor_util.py +960 -0
- diffusion_policy/model/diffusion/__init__.py +0 -0
- diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
- diffusion_policy/model/diffusion/conv1d_components.py +46 -0
- diffusion_policy/model/diffusion/ema_model.py +88 -0
- diffusion_policy/model/diffusion/mask_generator.py +221 -0
- diffusion_policy/model/diffusion/positional_embedding.py +17 -0
- diffusion_policy/model/vision/__init__.py +0 -0
- diffusion_policy/model/vision/crop_randomizer.py +288 -0
- diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
- diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
- diffusion_policy/policy/__init__.py +0 -0
- diffusion_policy/policy/base_image_policy.py +27 -0
- diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
- diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
- diffusion_policy/policy/parl_action_optimizer.py +675 -0
- diffusion_policy/workspace/__init__.py +0 -0
- diffusion_policy/workspace/base_workspace.py +146 -0
- diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
- solarian_diffusion/__init__.py +32 -0
- solarian_diffusion/config.py +31 -0
- solarian_diffusion/policy_server.py +316 -0
- solarian_diffusion/rtc.py +272 -0
- solarian_diffusion/service.py +16 -0
- solarian_diffusion/trainer.py +213 -0
- solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
- solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
- solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
- solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
- solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
- solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
- solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
- umi/__init__.py +0 -0
- umi/common/__init__.py +0 -0
- umi/common/pose_util.py +115 -0
|
@@ -0,0 +1,525 @@
|
|
|
1
|
+
import copy
|
|
2
|
+
|
|
3
|
+
import timm
|
|
4
|
+
import math
|
|
5
|
+
import torch
|
|
6
|
+
import torch.nn as nn
|
|
7
|
+
import torch.nn.functional as F
|
|
8
|
+
import torchvision
|
|
9
|
+
import logging
|
|
10
|
+
|
|
11
|
+
from diffusion_policy.model.common.module_attr_mixin import ModuleAttrMixin
|
|
12
|
+
|
|
13
|
+
from diffusion_policy.common.pytorch_util import replace_submodules
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _expand_vit_patch_embed_to_4ch(model: nn.Module) -> None:
|
|
19
|
+
"""Replace ``model.patch_embed.proj`` with a 4-input-channel conv.
|
|
20
|
+
|
|
21
|
+
The first three input-channel slices are copied verbatim from the
|
|
22
|
+
pretrained conv so the RGB path is bit-identical at initialization;
|
|
23
|
+
the new 4th input-channel slice is zero-initialized so the modified
|
|
24
|
+
network's output equals the original network's output on RGB-only
|
|
25
|
+
inputs at step 0. The optimizer then grows the 4th-channel weights
|
|
26
|
+
only when the mask carries gradient-reducing information.
|
|
27
|
+
|
|
28
|
+
Implements the "zero-init weight expansion" / identity-preserving-at-init
|
|
29
|
+
recipe used to bolt new modalities onto pretrained backbones.
|
|
30
|
+
"""
|
|
31
|
+
if not (hasattr(model, "patch_embed") and hasattr(model.patch_embed, "proj")):
|
|
32
|
+
raise RuntimeError(
|
|
33
|
+
"bbox_mode=mask_channel requires a ViT-style encoder with "
|
|
34
|
+
"`patch_embed.proj` (a Conv2d). Got a model of type "
|
|
35
|
+
f"{type(model).__name__} without that structure."
|
|
36
|
+
)
|
|
37
|
+
old_proj: nn.Conv2d = model.patch_embed.proj
|
|
38
|
+
if not isinstance(old_proj, nn.Conv2d):
|
|
39
|
+
raise RuntimeError(
|
|
40
|
+
f"patch_embed.proj is not a Conv2d (got {type(old_proj).__name__})."
|
|
41
|
+
)
|
|
42
|
+
if old_proj.in_channels != 3:
|
|
43
|
+
raise RuntimeError(
|
|
44
|
+
"Expected pretrained patch_embed.proj.in_channels == 3, got "
|
|
45
|
+
f"{old_proj.in_channels}. Refusing to overwrite — the encoder "
|
|
46
|
+
"may already have been surgically modified."
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
new_proj = nn.Conv2d(
|
|
50
|
+
in_channels=4,
|
|
51
|
+
out_channels=old_proj.out_channels,
|
|
52
|
+
kernel_size=old_proj.kernel_size,
|
|
53
|
+
stride=old_proj.stride,
|
|
54
|
+
padding=old_proj.padding,
|
|
55
|
+
dilation=old_proj.dilation,
|
|
56
|
+
groups=old_proj.groups,
|
|
57
|
+
bias=(old_proj.bias is not None),
|
|
58
|
+
)
|
|
59
|
+
with torch.no_grad():
|
|
60
|
+
new_proj.weight[:, :3].copy_(old_proj.weight)
|
|
61
|
+
new_proj.weight[:, 3:].zero_()
|
|
62
|
+
if old_proj.bias is not None:
|
|
63
|
+
new_proj.bias.copy_(old_proj.bias)
|
|
64
|
+
model.patch_embed.proj = new_proj
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def rasterize_bbox_to_mask(bbox: torch.Tensor, H: int, W: int) -> torch.Tensor:
|
|
68
|
+
"""Rasterize pixel-space bboxes to a binary mask.
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
bbox: tensor of shape ``(..., 4)`` with ``[x0, y0, x1, y1]`` in
|
|
72
|
+
pixel coordinates of the ``H x W`` image.
|
|
73
|
+
H: target mask height in pixels.
|
|
74
|
+
W: target mask width in pixels.
|
|
75
|
+
|
|
76
|
+
Returns:
|
|
77
|
+
Float tensor of shape ``(..., 1, H, W)`` with values in ``{0, 1}``.
|
|
78
|
+
Pixels strictly inside ``[x0, x1) x [y0, y1)`` are 1.
|
|
79
|
+
"""
|
|
80
|
+
leading_shape = bbox.shape[:-1]
|
|
81
|
+
flat = bbox.reshape(-1, 4)
|
|
82
|
+
N = flat.shape[0]
|
|
83
|
+
|
|
84
|
+
device = flat.device
|
|
85
|
+
# Use float32 for the comparison grid regardless of bbox dtype so we
|
|
86
|
+
# don't lose precision when the encoder is running under bf16/fp16.
|
|
87
|
+
xs = torch.arange(W, device=device, dtype=torch.float32).view(1, 1, 1, W)
|
|
88
|
+
ys = torch.arange(H, device=device, dtype=torch.float32).view(1, 1, H, 1)
|
|
89
|
+
|
|
90
|
+
flat_f = flat.to(torch.float32)
|
|
91
|
+
x0 = flat_f[:, 0].view(N, 1, 1, 1)
|
|
92
|
+
y0 = flat_f[:, 1].view(N, 1, 1, 1)
|
|
93
|
+
x1 = flat_f[:, 2].view(N, 1, 1, 1)
|
|
94
|
+
y1 = flat_f[:, 3].view(N, 1, 1, 1)
|
|
95
|
+
|
|
96
|
+
in_x = (xs >= x0) & (xs < x1)
|
|
97
|
+
in_y = (ys >= y0) & (ys < y1)
|
|
98
|
+
mask = (in_x & in_y).to(bbox.dtype)
|
|
99
|
+
return mask.reshape(*leading_shape, 1, H, W)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class MaskAwareImageTransform(nn.Module):
|
|
103
|
+
"""Apply geometric transforms to all channels, color transforms only to RGB.
|
|
104
|
+
|
|
105
|
+
Used when the input image has been augmented with extra non-RGB channels
|
|
106
|
+
(e.g. a binary bbox mask). RandomCrop / Resize keep the mask aligned with
|
|
107
|
+
the image; ColorJitter / Normalize would corrupt the mask if applied to
|
|
108
|
+
all channels, so they are restricted to the first three (RGB) channels.
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
def __init__(self, geometric: list, rgb_only: list, n_rgb: int = 3):
|
|
112
|
+
super().__init__()
|
|
113
|
+
self.geometric = nn.Sequential(*geometric) if geometric else nn.Identity()
|
|
114
|
+
self.rgb_only = nn.Sequential(*rgb_only) if rgb_only else nn.Identity()
|
|
115
|
+
self.n_rgb = n_rgb
|
|
116
|
+
|
|
117
|
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
118
|
+
x = self.geometric(x)
|
|
119
|
+
if x.shape[1] <= self.n_rgb:
|
|
120
|
+
return self.rgb_only(x)
|
|
121
|
+
rgb = self.rgb_only(x[:, : self.n_rgb])
|
|
122
|
+
rest = x[:, self.n_rgb:]
|
|
123
|
+
return torch.cat([rgb, rest], dim=1)
|
|
124
|
+
|
|
125
|
+
class AttentionPool2d(nn.Module):
|
|
126
|
+
def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None):
|
|
127
|
+
super().__init__()
|
|
128
|
+
self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5)
|
|
129
|
+
self.k_proj = nn.Linear(embed_dim, embed_dim)
|
|
130
|
+
self.q_proj = nn.Linear(embed_dim, embed_dim)
|
|
131
|
+
self.v_proj = nn.Linear(embed_dim, embed_dim)
|
|
132
|
+
self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim)
|
|
133
|
+
self.num_heads = num_heads
|
|
134
|
+
|
|
135
|
+
def forward(self, x):
|
|
136
|
+
x = x.flatten(start_dim=2).permute(2, 0, 1) # NCHW -> (HW)NC
|
|
137
|
+
x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC
|
|
138
|
+
x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC
|
|
139
|
+
x, _ = F.multi_head_attention_forward(
|
|
140
|
+
query=x[:1], key=x, value=x,
|
|
141
|
+
embed_dim_to_check=x.shape[-1],
|
|
142
|
+
num_heads=self.num_heads,
|
|
143
|
+
q_proj_weight=self.q_proj.weight,
|
|
144
|
+
k_proj_weight=self.k_proj.weight,
|
|
145
|
+
v_proj_weight=self.v_proj.weight,
|
|
146
|
+
in_proj_weight=None,
|
|
147
|
+
in_proj_bias=torch.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]),
|
|
148
|
+
bias_k=None,
|
|
149
|
+
bias_v=None,
|
|
150
|
+
add_zero_attn=False,
|
|
151
|
+
dropout_p=0,
|
|
152
|
+
out_proj_weight=self.c_proj.weight,
|
|
153
|
+
out_proj_bias=self.c_proj.bias,
|
|
154
|
+
use_separate_proj_weight=True,
|
|
155
|
+
training=self.training,
|
|
156
|
+
need_weights=False
|
|
157
|
+
)
|
|
158
|
+
return x.squeeze(0)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class TimmObsEncoder(ModuleAttrMixin):
|
|
162
|
+
def __init__(self,
|
|
163
|
+
shape_meta: dict,
|
|
164
|
+
model_name: str,
|
|
165
|
+
pretrained: bool,
|
|
166
|
+
frozen: bool,
|
|
167
|
+
global_pool: str,
|
|
168
|
+
transforms: list,
|
|
169
|
+
# replace BatchNorm with GroupNorm
|
|
170
|
+
use_group_norm: bool=False,
|
|
171
|
+
# use single rgb model for all rgb inputs
|
|
172
|
+
share_rgb_model: bool=False,
|
|
173
|
+
# renormalize rgb input with imagenet normalization
|
|
174
|
+
# assuming input in [0,1]
|
|
175
|
+
imagenet_norm: bool=False,
|
|
176
|
+
feature_aggregation: str='spatial_embedding',
|
|
177
|
+
downsample_ratio: int=32,
|
|
178
|
+
position_encording: str='learnable',
|
|
179
|
+
|
|
180
|
+
):
|
|
181
|
+
"""
|
|
182
|
+
Assumes rgb input: B,T,C,H,W
|
|
183
|
+
Assumes low_dim input: B,T,D
|
|
184
|
+
"""
|
|
185
|
+
super().__init__()
|
|
186
|
+
|
|
187
|
+
rgb_keys = list()
|
|
188
|
+
low_dim_keys = list()
|
|
189
|
+
key_model_map = nn.ModuleDict()
|
|
190
|
+
key_transform_map = nn.ModuleDict()
|
|
191
|
+
key_shape_map = dict()
|
|
192
|
+
|
|
193
|
+
assert global_pool == ''
|
|
194
|
+
model = timm.create_model(
|
|
195
|
+
model_name=model_name,
|
|
196
|
+
pretrained=pretrained,
|
|
197
|
+
global_pool=global_pool, # '' means no pooling
|
|
198
|
+
num_classes=0 # remove classification layer
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
if frozen:
|
|
202
|
+
assert pretrained
|
|
203
|
+
for param in model.parameters():
|
|
204
|
+
param.requires_grad = False
|
|
205
|
+
|
|
206
|
+
feature_dim = None
|
|
207
|
+
if model_name.startswith('resnet'):
|
|
208
|
+
# the last layer is nn.Identity() because num_classes is 0
|
|
209
|
+
# second last layer is AdaptivePool2d, which is also identity because global_pool is empty
|
|
210
|
+
if downsample_ratio == 32:
|
|
211
|
+
modules = list(model.children())[:-2]
|
|
212
|
+
model = torch.nn.Sequential(*modules)
|
|
213
|
+
feature_dim = 512
|
|
214
|
+
elif downsample_ratio == 16:
|
|
215
|
+
modules = list(model.children())[:-3]
|
|
216
|
+
model = torch.nn.Sequential(*modules)
|
|
217
|
+
feature_dim = 256
|
|
218
|
+
else:
|
|
219
|
+
raise NotImplementedError(f"Unsupported downsample_ratio: {downsample_ratio}")
|
|
220
|
+
elif model_name.startswith('convnext'):
|
|
221
|
+
# the last layer is nn.Identity() because num_classes is 0
|
|
222
|
+
# second last layer is AdaptivePool2d, which is also identity because global_pool is empty
|
|
223
|
+
if downsample_ratio == 32:
|
|
224
|
+
modules = list(model.children())[:-2]
|
|
225
|
+
model = torch.nn.Sequential(*modules)
|
|
226
|
+
feature_dim = 1024
|
|
227
|
+
else:
|
|
228
|
+
raise NotImplementedError(f"Unsupported downsample_ratio: {downsample_ratio}")
|
|
229
|
+
|
|
230
|
+
if use_group_norm and not pretrained:
|
|
231
|
+
model = replace_submodules(
|
|
232
|
+
root_module=model,
|
|
233
|
+
predicate=lambda x: isinstance(x, nn.BatchNorm2d),
|
|
234
|
+
func=lambda x: nn.GroupNorm(
|
|
235
|
+
num_groups=(x.num_features // 16) if (x.num_features % 16 == 0) else (x.num_features // 8),
|
|
236
|
+
num_channels=x.num_features)
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
image_shape = None
|
|
240
|
+
obs_shape_meta = shape_meta['obs']
|
|
241
|
+
for key, attr in obs_shape_meta.items():
|
|
242
|
+
shape = tuple(attr['shape'])
|
|
243
|
+
type = attr.get('type', 'low_dim')
|
|
244
|
+
if type == 'rgb':
|
|
245
|
+
assert image_shape is None or image_shape == shape[1:]
|
|
246
|
+
image_shape = shape[1:]
|
|
247
|
+
|
|
248
|
+
# ---- bbox -> mask-channel routing -----------------------------------
|
|
249
|
+
# Scan low_dim obs entries for `bbox_mode == 'mask_channel'`. Each such
|
|
250
|
+
# entry is rasterized into a binary mask and concatenated as a 4th
|
|
251
|
+
# channel onto a target RGB key inside forward(). The bbox itself is
|
|
252
|
+
# removed from the low-dim concat path so it doesn't get fed twice.
|
|
253
|
+
# `bbox_mask_map`: rgb_key -> bbox_key (the bbox routed onto that rgb).
|
|
254
|
+
bbox_mask_map = dict()
|
|
255
|
+
bbox_mask_horizon = dict() # rgb_key -> expected matching horizon
|
|
256
|
+
for key, attr in obs_shape_meta.items():
|
|
257
|
+
if attr.get('type', 'low_dim') != 'low_dim':
|
|
258
|
+
continue
|
|
259
|
+
if attr.get('bbox_mode', 'low_dim') != 'mask_channel':
|
|
260
|
+
continue
|
|
261
|
+
shape = tuple(attr['shape'])
|
|
262
|
+
if shape != (4,):
|
|
263
|
+
raise RuntimeError(
|
|
264
|
+
f"bbox_mode=mask_channel requires shape [4] (x0,y0,x1,y1) "
|
|
265
|
+
f"for key '{key}', got {shape}.")
|
|
266
|
+
target_rgb_key = attr.get('target_rgb_key')
|
|
267
|
+
if target_rgb_key is None:
|
|
268
|
+
rgb_candidates = [
|
|
269
|
+
k for k, a in obs_shape_meta.items()
|
|
270
|
+
if a.get('type', 'low_dim') == 'rgb']
|
|
271
|
+
if len(rgb_candidates) != 1:
|
|
272
|
+
raise RuntimeError(
|
|
273
|
+
f"bbox key '{key}' with bbox_mode=mask_channel did not "
|
|
274
|
+
f"specify `target_rgb_key` and there are "
|
|
275
|
+
f"{len(rgb_candidates)} candidate rgb keys; please set "
|
|
276
|
+
f"`target_rgb_key` explicitly in shape_meta.")
|
|
277
|
+
target_rgb_key = rgb_candidates[0]
|
|
278
|
+
if target_rgb_key not in obs_shape_meta or \
|
|
279
|
+
obs_shape_meta[target_rgb_key].get('type') != 'rgb':
|
|
280
|
+
raise RuntimeError(
|
|
281
|
+
f"bbox key '{key}' targets rgb_key='{target_rgb_key}' "
|
|
282
|
+
f"which is not an rgb obs entry.")
|
|
283
|
+
if target_rgb_key in bbox_mask_map:
|
|
284
|
+
raise RuntimeError(
|
|
285
|
+
f"Multiple bbox keys are routed to rgb_key="
|
|
286
|
+
f"'{target_rgb_key}'. Only one bbox mask channel per "
|
|
287
|
+
f"image is supported.")
|
|
288
|
+
rgb_horizon = int(obs_shape_meta[target_rgb_key]['horizon'])
|
|
289
|
+
bbox_horizon = int(attr['horizon'])
|
|
290
|
+
if bbox_horizon != rgb_horizon:
|
|
291
|
+
raise RuntimeError(
|
|
292
|
+
f"bbox '{key}' (horizon={bbox_horizon}) and target image "
|
|
293
|
+
f"'{target_rgb_key}' (horizon={rgb_horizon}) must share "
|
|
294
|
+
f"the same horizon when bbox_mode=mask_channel.")
|
|
295
|
+
bbox_mask_map[target_rgb_key] = key
|
|
296
|
+
bbox_mask_horizon[target_rgb_key] = rgb_horizon
|
|
297
|
+
|
|
298
|
+
if transforms is not None and not isinstance(transforms[0], torch.nn.Module):
|
|
299
|
+
assert transforms[0].type == 'RandomCrop'
|
|
300
|
+
ratio = transforms[0].ratio
|
|
301
|
+
transforms = [
|
|
302
|
+
torchvision.transforms.RandomCrop(size=int(image_shape[0] * ratio)),
|
|
303
|
+
torchvision.transforms.Resize(size=image_shape[0], antialias=True)
|
|
304
|
+
] + transforms[1:]
|
|
305
|
+
# The two leading transforms (RandomCrop, Resize) are geometric and
|
|
306
|
+
# are safe on arbitrary channel counts; everything after them is RGB-
|
|
307
|
+
# only (ColorJitter uses HSV, ImageNet Normalize assumes 3 channels).
|
|
308
|
+
geometric_transforms = transforms[:2] if transforms is not None else []
|
|
309
|
+
color_transforms = transforms[2:] if transforms is not None else []
|
|
310
|
+
rgb_only_transform = (
|
|
311
|
+
nn.Identity() if transforms is None
|
|
312
|
+
else torch.nn.Sequential(*transforms)
|
|
313
|
+
)
|
|
314
|
+
|
|
315
|
+
for key, attr in obs_shape_meta.items():
|
|
316
|
+
shape = tuple(attr['shape'])
|
|
317
|
+
type = attr.get('type', 'low_dim')
|
|
318
|
+
key_shape_map[key] = shape
|
|
319
|
+
if type == 'rgb':
|
|
320
|
+
rgb_keys.append(key)
|
|
321
|
+
|
|
322
|
+
this_model = model if share_rgb_model else copy.deepcopy(model)
|
|
323
|
+
if key in bbox_mask_map:
|
|
324
|
+
# Expand the patch embedding to accept the mask channel.
|
|
325
|
+
# Zero-init the new slice so the model output is identical
|
|
326
|
+
# to the pretrained model at step 0 (see header docstring).
|
|
327
|
+
if share_rgb_model:
|
|
328
|
+
raise RuntimeError(
|
|
329
|
+
"bbox_mode=mask_channel is incompatible with "
|
|
330
|
+
"share_rgb_model=True (the patch_embed surgery "
|
|
331
|
+
"would be applied multiple times to the same "
|
|
332
|
+
"tensor). Disable share_rgb_model or unset the "
|
|
333
|
+
"mask routing for this key.")
|
|
334
|
+
_expand_vit_patch_embed_to_4ch(this_model)
|
|
335
|
+
this_transform = MaskAwareImageTransform(
|
|
336
|
+
geometric=geometric_transforms,
|
|
337
|
+
rgb_only=color_transforms,
|
|
338
|
+
)
|
|
339
|
+
else:
|
|
340
|
+
this_transform = rgb_only_transform
|
|
341
|
+
key_model_map[key] = this_model
|
|
342
|
+
key_transform_map[key] = this_transform
|
|
343
|
+
elif type == 'low_dim':
|
|
344
|
+
if attr.get('ignore_by_policy', False):
|
|
345
|
+
continue
|
|
346
|
+
if attr.get('bbox_mode', 'low_dim') == 'mask_channel':
|
|
347
|
+
# consumed via the rgb path, not as a flat feature.
|
|
348
|
+
continue
|
|
349
|
+
low_dim_keys.append(key)
|
|
350
|
+
else:
|
|
351
|
+
raise RuntimeError(f"Unsupported obs type: {type}")
|
|
352
|
+
|
|
353
|
+
feature_map_shape = [x // downsample_ratio for x in image_shape]
|
|
354
|
+
|
|
355
|
+
rgb_keys = sorted(rgb_keys)
|
|
356
|
+
low_dim_keys = sorted(low_dim_keys)
|
|
357
|
+
print('rgb keys: ', rgb_keys)
|
|
358
|
+
print('low_dim_keys keys:', low_dim_keys)
|
|
359
|
+
|
|
360
|
+
self.model_name = model_name
|
|
361
|
+
self.shape_meta = shape_meta
|
|
362
|
+
self.key_model_map = key_model_map
|
|
363
|
+
self.key_transform_map = key_transform_map
|
|
364
|
+
self.share_rgb_model = share_rgb_model
|
|
365
|
+
self.rgb_keys = rgb_keys
|
|
366
|
+
self.low_dim_keys = low_dim_keys
|
|
367
|
+
self.key_shape_map = key_shape_map
|
|
368
|
+
self.feature_aggregation = feature_aggregation
|
|
369
|
+
# rgb_key -> bbox_key. Empty dict means no mask routing is active and
|
|
370
|
+
# forward() behaves exactly as before. Persisted on the encoder so
|
|
371
|
+
# downstream code (eval, loggers, debug tools) can introspect the
|
|
372
|
+
# bbox integration mode without re-parsing shape_meta.
|
|
373
|
+
self.bbox_mask_map = bbox_mask_map
|
|
374
|
+
if model_name.startswith('vit'):
|
|
375
|
+
# assert self.feature_aggregation is None # vit uses the CLS token
|
|
376
|
+
if self.feature_aggregation == 'all_tokens':
|
|
377
|
+
# Use all tokens from ViT
|
|
378
|
+
pass
|
|
379
|
+
elif self.feature_aggregation is not None:
|
|
380
|
+
logger.warn(f'vit will use the CLS token. feature_aggregation ({self.feature_aggregation}) is ignored!')
|
|
381
|
+
self.feature_aggregation = None
|
|
382
|
+
|
|
383
|
+
if self.feature_aggregation == 'soft_attention':
|
|
384
|
+
self.attention = nn.Sequential(
|
|
385
|
+
nn.Linear(feature_dim, 1, bias=False),
|
|
386
|
+
nn.Softmax(dim=1)
|
|
387
|
+
)
|
|
388
|
+
elif self.feature_aggregation == 'spatial_embedding':
|
|
389
|
+
self.spatial_embedding = torch.nn.Parameter(torch.randn(feature_map_shape[0] * feature_map_shape[1], feature_dim))
|
|
390
|
+
elif self.feature_aggregation == 'transformer':
|
|
391
|
+
if position_encording == 'learnable':
|
|
392
|
+
self.position_embedding = torch.nn.Parameter(torch.randn(feature_map_shape[0] * feature_map_shape[1] + 1, feature_dim))
|
|
393
|
+
elif position_encording == 'sinusoidal':
|
|
394
|
+
num_features = feature_map_shape[0] * feature_map_shape[1] + 1
|
|
395
|
+
self.position_embedding = torch.zeros(num_features, feature_dim)
|
|
396
|
+
position = torch.arange(0, num_features, dtype=torch.float).unsqueeze(1)
|
|
397
|
+
div_term = torch.exp(torch.arange(0, feature_dim, 2).float() * (-math.log(2 * num_features) / feature_dim))
|
|
398
|
+
self.position_embedding[:, 0::2] = torch.sin(position * div_term)
|
|
399
|
+
self.position_embedding[:, 1::2] = torch.cos(position * div_term)
|
|
400
|
+
self.aggregation_transformer = nn.TransformerEncoder(
|
|
401
|
+
encoder_layer=nn.TransformerEncoderLayer(d_model=feature_dim, nhead=4),
|
|
402
|
+
num_layers=4)
|
|
403
|
+
elif self.feature_aggregation == 'attention_pool_2d':
|
|
404
|
+
self.attention_pool_2d = AttentionPool2d(
|
|
405
|
+
spacial_dim=feature_map_shape[0],
|
|
406
|
+
embed_dim=feature_dim,
|
|
407
|
+
num_heads=feature_dim // 64,
|
|
408
|
+
output_dim=feature_dim
|
|
409
|
+
)
|
|
410
|
+
logger.info(
|
|
411
|
+
"number of parameters: %e", sum(p.numel() for p in self.parameters())
|
|
412
|
+
)
|
|
413
|
+
|
|
414
|
+
def aggregate_feature(self, feature):
|
|
415
|
+
if self.model_name.startswith('vit'):
|
|
416
|
+
assert self.feature_aggregation is None # vit uses the CLS token
|
|
417
|
+
return feature[:, 0, :]
|
|
418
|
+
|
|
419
|
+
# resnet
|
|
420
|
+
assert len(feature.shape) == 4
|
|
421
|
+
if self.feature_aggregation == 'attention_pool_2d':
|
|
422
|
+
return self.attention_pool_2d(feature)
|
|
423
|
+
|
|
424
|
+
feature = torch.flatten(feature, start_dim=-2) # B, 512, 7*7
|
|
425
|
+
feature = torch.transpose(feature, 1, 2) # B, 7*7, 512
|
|
426
|
+
|
|
427
|
+
if self.feature_aggregation == 'avg':
|
|
428
|
+
return torch.mean(feature, dim=[1])
|
|
429
|
+
elif self.feature_aggregation == 'max':
|
|
430
|
+
return torch.amax(feature, dim=[1])
|
|
431
|
+
elif self.feature_aggregation == 'soft_attention':
|
|
432
|
+
weight = self.attention(feature)
|
|
433
|
+
return torch.sum(feature * weight, dim=1)
|
|
434
|
+
elif self.feature_aggregation == 'spatial_embedding':
|
|
435
|
+
return torch.mean(feature * self.spatial_embedding, dim=1)
|
|
436
|
+
elif self.feature_aggregation == 'transformer':
|
|
437
|
+
zero_feature = torch.zeros(feature.shape[0], 1, feature.shape[-1], device=feature.device)
|
|
438
|
+
if self.position_embedding.device != feature.device:
|
|
439
|
+
self.position_embedding = self.position_embedding.to(feature.device)
|
|
440
|
+
feature_with_pos_embedding = torch.concat([zero_feature, feature], dim=1) + self.position_embedding
|
|
441
|
+
feature_output = self.aggregation_transformer(feature_with_pos_embedding)
|
|
442
|
+
return feature_output[:, 0]
|
|
443
|
+
else:
|
|
444
|
+
assert self.feature_aggregation is None
|
|
445
|
+
return feature
|
|
446
|
+
|
|
447
|
+
def forward(self, obs_dict):
|
|
448
|
+
features = list()
|
|
449
|
+
batch_size = next(iter(obs_dict.values())).shape[0]
|
|
450
|
+
|
|
451
|
+
# process rgb input
|
|
452
|
+
for key in self.rgb_keys:
|
|
453
|
+
img = obs_dict[key]
|
|
454
|
+
B, T = img.shape[:2]
|
|
455
|
+
assert B == batch_size
|
|
456
|
+
assert img.shape[2:] == self.key_shape_map[key]
|
|
457
|
+
|
|
458
|
+
if key in self.bbox_mask_map:
|
|
459
|
+
# Rasterize the routed bbox into a (B, T, 1, H, W) binary
|
|
460
|
+
# mask and concat as a 4th channel. Geometric transforms in
|
|
461
|
+
# MaskAwareImageTransform then crop/resize image+mask
|
|
462
|
+
# together, keeping the bbox geometry consistent with the
|
|
463
|
+
# image content the ViT sees.
|
|
464
|
+
bbox_key = self.bbox_mask_map[key]
|
|
465
|
+
bbox = obs_dict[bbox_key]
|
|
466
|
+
assert bbox.shape[0] == batch_size, (
|
|
467
|
+
f"bbox '{bbox_key}' batch={bbox.shape[0]} mismatches "
|
|
468
|
+
f"image '{key}' batch={batch_size}")
|
|
469
|
+
assert bbox.shape[1] == T, (
|
|
470
|
+
f"bbox '{bbox_key}' horizon={bbox.shape[1]} mismatches "
|
|
471
|
+
f"image '{key}' horizon={T}")
|
|
472
|
+
assert bbox.shape[2:] == (4,), (
|
|
473
|
+
f"bbox '{bbox_key}' expected shape [4], got "
|
|
474
|
+
f"{tuple(bbox.shape[2:])}")
|
|
475
|
+
_, _, _, H, W = img.shape
|
|
476
|
+
mask = rasterize_bbox_to_mask(bbox, H=H, W=W).to(img.dtype)
|
|
477
|
+
img = torch.cat([img, mask], dim=2)
|
|
478
|
+
|
|
479
|
+
img = img.reshape(B*T, *img.shape[2:])
|
|
480
|
+
img = self.key_transform_map[key](img)
|
|
481
|
+
raw_feature = self.key_model_map[key](img)
|
|
482
|
+
feature = self.aggregate_feature(raw_feature)
|
|
483
|
+
assert len(feature.shape) == 2 and feature.shape[0] == B * T
|
|
484
|
+
features.append(feature.reshape(B, -1))
|
|
485
|
+
|
|
486
|
+
# process lowdim input
|
|
487
|
+
for key in self.low_dim_keys:
|
|
488
|
+
data = obs_dict[key]
|
|
489
|
+
B, T = data.shape[:2]
|
|
490
|
+
assert B == batch_size
|
|
491
|
+
assert data.shape[2:] == self.key_shape_map[key]
|
|
492
|
+
features.append(data.reshape(B, -1))
|
|
493
|
+
|
|
494
|
+
# concatenate all features
|
|
495
|
+
result = torch.cat(features, dim=-1)
|
|
496
|
+
|
|
497
|
+
return result
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
@torch.no_grad()
|
|
501
|
+
def output_shape(self):
|
|
502
|
+
example_obs_dict = dict()
|
|
503
|
+
obs_shape_meta = self.shape_meta['obs']
|
|
504
|
+
for key, attr in obs_shape_meta.items():
|
|
505
|
+
shape = tuple(attr['shape'])
|
|
506
|
+
this_obs = torch.zeros(
|
|
507
|
+
(1, attr['horizon']) + shape,
|
|
508
|
+
dtype=self.dtype,
|
|
509
|
+
device=self.device)
|
|
510
|
+
example_obs_dict[key] = this_obs
|
|
511
|
+
example_output = self.forward(example_obs_dict)
|
|
512
|
+
assert len(example_output.shape) == 2
|
|
513
|
+
assert example_output.shape[0] == 1
|
|
514
|
+
|
|
515
|
+
return example_output.shape
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
if __name__=='__main__':
|
|
519
|
+
timm_obs_encoder = TimmObsEncoder(
|
|
520
|
+
shape_meta=None,
|
|
521
|
+
model_name='resnet18.a1_in1k',
|
|
522
|
+
pretrained=False,
|
|
523
|
+
global_pool='',
|
|
524
|
+
transforms=None
|
|
525
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from typing import Dict
|
|
2
|
+
import torch
|
|
3
|
+
import torch.nn as nn
|
|
4
|
+
from diffusion_policy.model.common.module_attr_mixin import ModuleAttrMixin
|
|
5
|
+
from diffusion_policy.model.common.normalizer import LinearNormalizer
|
|
6
|
+
|
|
7
|
+
class BaseImagePolicy(ModuleAttrMixin):
|
|
8
|
+
# init accepts keyword argument shape_meta, see config/task/*_image.yaml
|
|
9
|
+
|
|
10
|
+
def predict_action(self, obs_dict: Dict[str, torch.Tensor], fixed_action_prefix: torch.Tensor=None) -> Dict[str, torch.Tensor]:
|
|
11
|
+
"""
|
|
12
|
+
obs_dict:
|
|
13
|
+
str: B,To,*
|
|
14
|
+
fixed_action_prefix:
|
|
15
|
+
B, Tp, Da
|
|
16
|
+
return: B,Ta,Da
|
|
17
|
+
"""
|
|
18
|
+
raise NotImplementedError()
|
|
19
|
+
|
|
20
|
+
# reset state for stateful policies
|
|
21
|
+
def reset(self):
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
# ========== training ===========
|
|
25
|
+
# no standard training interface except setting normalizer
|
|
26
|
+
def set_normalizer(self, normalizer: LinearNormalizer):
|
|
27
|
+
raise NotImplementedError()
|