solarian-diffusion 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffusion_policy/__init__.py +0 -0
- diffusion_policy/codecs/__init__.py +0 -0
- diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
- diffusion_policy/common/__init__.py +0 -0
- diffusion_policy/common/checkpoint_util.py +64 -0
- diffusion_policy/common/json_logger.py +117 -0
- diffusion_policy/common/normalize_util.py +251 -0
- diffusion_policy/common/pose_repr_util.py +112 -0
- diffusion_policy/common/pytorch_util.py +82 -0
- diffusion_policy/common/q_critic_util.py +135 -0
- diffusion_policy/common/replay_buffer.py +672 -0
- diffusion_policy/common/sampler.py +204 -0
- diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
- diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
- diffusion_policy/config/legacy/task/can_image.yaml +64 -0
- diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
- diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
- diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
- diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
- diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
- diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
- diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
- diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
- diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
- diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
- diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
- diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
- diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
- diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
- diffusion_policy/config/legacy/task/square_image.yaml +64 -0
- diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
- diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
- diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
- diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
- diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
- diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
- diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
- diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
- diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
- diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
- diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
- diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
- diffusion_policy/config/legacy/task/umi.yaml +45 -0
- diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
- diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
- diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
- diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
- diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
- diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
- diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
- diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
- diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
- diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
- diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
- diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
- diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
- diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
- diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
- diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
- diffusion_policy/config/rl_annotate.yaml +265 -0
- diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
- diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
- diffusion_policy/config/rl_annotate_v2.yaml +141 -0
- diffusion_policy/config/rl_annotate_v3.yaml +165 -0
- diffusion_policy/config/task/lift.yaml +89 -0
- diffusion_policy/config/task/square.yaml +92 -0
- diffusion_policy/config/task/tool_hang.yaml +93 -0
- diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
- diffusion_policy/config/task/umi.yaml +105 -0
- diffusion_policy/config/task/umi_bimanual.yaml +135 -0
- diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
- diffusion_policy/config/task/umi_image.yaml +36 -0
- diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
- diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
- diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
- diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
- diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
- diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
- diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
- diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
- diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
- diffusion_policy/config/train_iql_umi.yaml +186 -0
- diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
- diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
- diffusion_policy/dataset/__init__.py +0 -0
- diffusion_policy/dataset/base_dataset.py +76 -0
- diffusion_policy/dataset/umi_dataset.py +652 -0
- diffusion_policy/env_runner/__init__.py +0 -0
- diffusion_policy/env_runner/base_image_runner.py +9 -0
- diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
- diffusion_policy/model/__init__.py +0 -0
- diffusion_policy/model/common/__init__.py +0 -0
- diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
- diffusion_policy/model/common/lr_scheduler.py +46 -0
- diffusion_policy/model/common/module_attr_mixin.py +14 -0
- diffusion_policy/model/common/normalizer.py +353 -0
- diffusion_policy/model/common/tensor_util.py +960 -0
- diffusion_policy/model/diffusion/__init__.py +0 -0
- diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
- diffusion_policy/model/diffusion/conv1d_components.py +46 -0
- diffusion_policy/model/diffusion/ema_model.py +88 -0
- diffusion_policy/model/diffusion/mask_generator.py +221 -0
- diffusion_policy/model/diffusion/positional_embedding.py +17 -0
- diffusion_policy/model/vision/__init__.py +0 -0
- diffusion_policy/model/vision/crop_randomizer.py +288 -0
- diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
- diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
- diffusion_policy/policy/__init__.py +0 -0
- diffusion_policy/policy/base_image_policy.py +27 -0
- diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
- diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
- diffusion_policy/policy/parl_action_optimizer.py +675 -0
- diffusion_policy/workspace/__init__.py +0 -0
- diffusion_policy/workspace/base_workspace.py +146 -0
- diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
- solarian_diffusion/__init__.py +32 -0
- solarian_diffusion/config.py +31 -0
- solarian_diffusion/policy_server.py +316 -0
- solarian_diffusion/rtc.py +272 -0
- solarian_diffusion/service.py +16 -0
- solarian_diffusion/trainer.py +213 -0
- solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
- solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
- solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
- solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
- solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
- solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
- solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
- umi/__init__.py +0 -0
- umi/common/__init__.py +0 -0
- umi/common/pose_util.py +115 -0
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
import torch.nn as nn
|
|
3
|
+
import torchvision.transforms.functional as ttf
|
|
4
|
+
import diffusion_policy.model.common.tensor_util as tu
|
|
5
|
+
|
|
6
|
+
class CropRandomizer(nn.Module):
|
|
7
|
+
"""
|
|
8
|
+
Randomly sample crops at input, and then average across crop features at output.
|
|
9
|
+
"""
|
|
10
|
+
def __init__(
|
|
11
|
+
self,
|
|
12
|
+
input_shape,
|
|
13
|
+
crop_height,
|
|
14
|
+
crop_width,
|
|
15
|
+
num_crops=1,
|
|
16
|
+
pos_enc=False,
|
|
17
|
+
):
|
|
18
|
+
"""
|
|
19
|
+
Args:
|
|
20
|
+
input_shape (tuple, list): shape of input (not including batch dimension)
|
|
21
|
+
crop_height (int): crop height
|
|
22
|
+
crop_width (int): crop width
|
|
23
|
+
num_crops (int): number of random crops to take
|
|
24
|
+
pos_enc (bool): if True, add 2 channels to the output to encode the spatial
|
|
25
|
+
location of the cropped pixels in the source image
|
|
26
|
+
"""
|
|
27
|
+
super().__init__()
|
|
28
|
+
|
|
29
|
+
assert len(input_shape) == 3 # (C, H, W)
|
|
30
|
+
assert crop_height < input_shape[1]
|
|
31
|
+
assert crop_width < input_shape[2]
|
|
32
|
+
|
|
33
|
+
self.input_shape = input_shape
|
|
34
|
+
self.crop_height = crop_height
|
|
35
|
+
self.crop_width = crop_width
|
|
36
|
+
self.num_crops = num_crops
|
|
37
|
+
self.pos_enc = pos_enc
|
|
38
|
+
|
|
39
|
+
def output_shape_in(self, input_shape=None):
|
|
40
|
+
"""
|
|
41
|
+
Function to compute output shape from inputs to this module. Corresponds to
|
|
42
|
+
the @forward_in operation, where raw inputs (usually observation modalities)
|
|
43
|
+
are passed in.
|
|
44
|
+
|
|
45
|
+
Args:
|
|
46
|
+
input_shape (iterable of int): shape of input. Does not include batch dimension.
|
|
47
|
+
Some modules may not need this argument, if their output does not depend
|
|
48
|
+
on the size of the input, or if they assume fixed size input.
|
|
49
|
+
|
|
50
|
+
Returns:
|
|
51
|
+
out_shape ([int]): list of integers corresponding to output shape
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
# outputs are shape (C, CH, CW), or maybe C + 2 if using position encoding, because
|
|
55
|
+
# the number of crops are reshaped into the batch dimension, increasing the batch
|
|
56
|
+
# size from B to B * N
|
|
57
|
+
out_c = self.input_shape[0] + 2 if self.pos_enc else self.input_shape[0]
|
|
58
|
+
return [out_c, self.crop_height, self.crop_width]
|
|
59
|
+
|
|
60
|
+
def output_shape_out(self, input_shape=None):
|
|
61
|
+
"""
|
|
62
|
+
Function to compute output shape from inputs to this module. Corresponds to
|
|
63
|
+
the @forward_out operation, where processed inputs (usually encoded observation
|
|
64
|
+
modalities) are passed in.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
input_shape (iterable of int): shape of input. Does not include batch dimension.
|
|
68
|
+
Some modules may not need this argument, if their output does not depend
|
|
69
|
+
on the size of the input, or if they assume fixed size input.
|
|
70
|
+
|
|
71
|
+
Returns:
|
|
72
|
+
out_shape ([int]): list of integers corresponding to output shape
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
# since the forward_out operation splits [B * N, ...] -> [B, N, ...]
|
|
76
|
+
# and then pools to result in [B, ...], only the batch dimension changes,
|
|
77
|
+
# and so the other dimensions retain their shape.
|
|
78
|
+
return list(input_shape)
|
|
79
|
+
|
|
80
|
+
def forward_in(self, inputs):
|
|
81
|
+
"""
|
|
82
|
+
Samples N random crops for each input in the batch, and then reshapes
|
|
83
|
+
inputs to [B * N, ...].
|
|
84
|
+
"""
|
|
85
|
+
assert len(inputs.shape) >= 3 # must have at least (C, H, W) dimensions
|
|
86
|
+
if self.training:
|
|
87
|
+
# generate random crops
|
|
88
|
+
out, _ = sample_random_image_crops(
|
|
89
|
+
images=inputs,
|
|
90
|
+
crop_height=self.crop_height,
|
|
91
|
+
crop_width=self.crop_width,
|
|
92
|
+
num_crops=self.num_crops,
|
|
93
|
+
pos_enc=self.pos_enc,
|
|
94
|
+
)
|
|
95
|
+
# [B, N, ...] -> [B * N, ...]
|
|
96
|
+
return tu.join_dimensions(out, 0, 1)
|
|
97
|
+
else:
|
|
98
|
+
# take center crop during eval
|
|
99
|
+
out = ttf.center_crop(img=inputs, output_size=(
|
|
100
|
+
self.crop_height, self.crop_width))
|
|
101
|
+
if self.num_crops > 1:
|
|
102
|
+
B,C,H,W = out.shape
|
|
103
|
+
out = out.unsqueeze(1).expand(B,self.num_crops,C,H,W).reshape(-1,C,H,W)
|
|
104
|
+
# [B * N, ...]
|
|
105
|
+
return out
|
|
106
|
+
|
|
107
|
+
def forward_out(self, inputs):
|
|
108
|
+
"""
|
|
109
|
+
Splits the outputs from shape [B * N, ...] -> [B, N, ...] and then average across N
|
|
110
|
+
to result in shape [B, ...] to make sure the network output is consistent with
|
|
111
|
+
what would have happened if there were no randomization.
|
|
112
|
+
"""
|
|
113
|
+
if self.num_crops <= 1:
|
|
114
|
+
return inputs
|
|
115
|
+
else:
|
|
116
|
+
batch_size = (inputs.shape[0] // self.num_crops)
|
|
117
|
+
out = tu.reshape_dimensions(inputs, begin_axis=0, end_axis=0,
|
|
118
|
+
target_dims=(batch_size, self.num_crops))
|
|
119
|
+
return out.mean(dim=1)
|
|
120
|
+
|
|
121
|
+
def forward(self, inputs):
|
|
122
|
+
return self.forward_in(inputs)
|
|
123
|
+
|
|
124
|
+
def __repr__(self):
|
|
125
|
+
"""Pretty print network."""
|
|
126
|
+
header = '{}'.format(str(self.__class__.__name__))
|
|
127
|
+
msg = header + "(input_shape={}, crop_size=[{}, {}], num_crops={})".format(
|
|
128
|
+
self.input_shape, self.crop_height, self.crop_width, self.num_crops)
|
|
129
|
+
return msg
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def crop_image_from_indices(images, crop_indices, crop_height, crop_width):
|
|
133
|
+
"""
|
|
134
|
+
Crops images at the locations specified by @crop_indices. Crops will be
|
|
135
|
+
taken across all channels.
|
|
136
|
+
|
|
137
|
+
Args:
|
|
138
|
+
images (torch.Tensor): batch of images of shape [..., C, H, W]
|
|
139
|
+
|
|
140
|
+
crop_indices (torch.Tensor): batch of indices of shape [..., N, 2] where
|
|
141
|
+
N is the number of crops to take per image and each entry corresponds
|
|
142
|
+
to the pixel height and width of where to take the crop. Note that
|
|
143
|
+
the indices can also be of shape [..., 2] if only 1 crop should
|
|
144
|
+
be taken per image. Leading dimensions must be consistent with
|
|
145
|
+
@images argument. Each index specifies the top left of the crop.
|
|
146
|
+
Values must be in range [0, H - CH - 1] x [0, W - CW - 1] where
|
|
147
|
+
H and W are the height and width of @images and CH and CW are
|
|
148
|
+
@crop_height and @crop_width.
|
|
149
|
+
|
|
150
|
+
crop_height (int): height of crop to take
|
|
151
|
+
|
|
152
|
+
crop_width (int): width of crop to take
|
|
153
|
+
|
|
154
|
+
Returns:
|
|
155
|
+
crops (torch.Tesnor): cropped images of shape [..., C, @crop_height, @crop_width]
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
# make sure length of input shapes is consistent
|
|
159
|
+
assert crop_indices.shape[-1] == 2
|
|
160
|
+
ndim_im_shape = len(images.shape)
|
|
161
|
+
ndim_indices_shape = len(crop_indices.shape)
|
|
162
|
+
assert (ndim_im_shape == ndim_indices_shape + 1) or (ndim_im_shape == ndim_indices_shape + 2)
|
|
163
|
+
|
|
164
|
+
# maybe pad so that @crop_indices is shape [..., N, 2]
|
|
165
|
+
is_padded = False
|
|
166
|
+
if ndim_im_shape == ndim_indices_shape + 2:
|
|
167
|
+
crop_indices = crop_indices.unsqueeze(-2)
|
|
168
|
+
is_padded = True
|
|
169
|
+
|
|
170
|
+
# make sure leading dimensions between images and indices are consistent
|
|
171
|
+
assert images.shape[:-3] == crop_indices.shape[:-2]
|
|
172
|
+
|
|
173
|
+
device = images.device
|
|
174
|
+
image_c, image_h, image_w = images.shape[-3:]
|
|
175
|
+
num_crops = crop_indices.shape[-2]
|
|
176
|
+
|
|
177
|
+
# make sure @crop_indices are in valid range
|
|
178
|
+
assert (crop_indices[..., 0] >= 0).all().item()
|
|
179
|
+
assert (crop_indices[..., 0] < (image_h - crop_height)).all().item()
|
|
180
|
+
assert (crop_indices[..., 1] >= 0).all().item()
|
|
181
|
+
assert (crop_indices[..., 1] < (image_w - crop_width)).all().item()
|
|
182
|
+
|
|
183
|
+
# convert each crop index (ch, cw) into a list of pixel indices that correspond to the entire window.
|
|
184
|
+
|
|
185
|
+
# 2D index array with columns [0, 1, ..., CH - 1] and shape [CH, CW]
|
|
186
|
+
crop_ind_grid_h = torch.arange(crop_height).to(device)
|
|
187
|
+
crop_ind_grid_h = tu.unsqueeze_expand_at(crop_ind_grid_h, size=crop_width, dim=-1)
|
|
188
|
+
# 2D index array with rows [0, 1, ..., CW - 1] and shape [CH, CW]
|
|
189
|
+
crop_ind_grid_w = torch.arange(crop_width).to(device)
|
|
190
|
+
crop_ind_grid_w = tu.unsqueeze_expand_at(crop_ind_grid_w, size=crop_height, dim=0)
|
|
191
|
+
# combine into shape [CH, CW, 2]
|
|
192
|
+
crop_in_grid = torch.cat((crop_ind_grid_h.unsqueeze(-1), crop_ind_grid_w.unsqueeze(-1)), dim=-1)
|
|
193
|
+
|
|
194
|
+
# Add above grid with the offset index of each sampled crop to get 2d indices for each crop.
|
|
195
|
+
# After broadcasting, this will be shape [..., N, CH, CW, 2] and each crop has a [CH, CW, 2]
|
|
196
|
+
# shape array that tells us which pixels from the corresponding source image to grab.
|
|
197
|
+
grid_reshape = [1] * len(crop_indices.shape[:-1]) + [crop_height, crop_width, 2]
|
|
198
|
+
all_crop_inds = crop_indices.unsqueeze(-2).unsqueeze(-2) + crop_in_grid.reshape(grid_reshape)
|
|
199
|
+
|
|
200
|
+
# For using @torch.gather, convert to flat indices from 2D indices, and also
|
|
201
|
+
# repeat across the channel dimension. To get flat index of each pixel to grab for
|
|
202
|
+
# each sampled crop, we just use the mapping: ind = h_ind * @image_w + w_ind
|
|
203
|
+
all_crop_inds = all_crop_inds[..., 0] * image_w + all_crop_inds[..., 1] # shape [..., N, CH, CW]
|
|
204
|
+
all_crop_inds = tu.unsqueeze_expand_at(all_crop_inds, size=image_c, dim=-3) # shape [..., N, C, CH, CW]
|
|
205
|
+
all_crop_inds = tu.flatten(all_crop_inds, begin_axis=-2) # shape [..., N, C, CH * CW]
|
|
206
|
+
|
|
207
|
+
# Repeat and flatten the source images -> [..., N, C, H * W] and then use gather to index with crop pixel inds
|
|
208
|
+
images_to_crop = tu.unsqueeze_expand_at(images, size=num_crops, dim=-4)
|
|
209
|
+
images_to_crop = tu.flatten(images_to_crop, begin_axis=-2)
|
|
210
|
+
crops = torch.gather(images_to_crop, dim=-1, index=all_crop_inds)
|
|
211
|
+
# [..., N, C, CH * CW] -> [..., N, C, CH, CW]
|
|
212
|
+
reshape_axis = len(crops.shape) - 1
|
|
213
|
+
crops = tu.reshape_dimensions(crops, begin_axis=reshape_axis, end_axis=reshape_axis,
|
|
214
|
+
target_dims=(crop_height, crop_width))
|
|
215
|
+
|
|
216
|
+
if is_padded:
|
|
217
|
+
# undo padding -> [..., C, CH, CW]
|
|
218
|
+
crops = crops.squeeze(-4)
|
|
219
|
+
return crops
|
|
220
|
+
|
|
221
|
+
def sample_random_image_crops(images, crop_height, crop_width, num_crops, pos_enc=False):
|
|
222
|
+
"""
|
|
223
|
+
For each image, randomly sample @num_crops crops of size (@crop_height, @crop_width), from
|
|
224
|
+
@images.
|
|
225
|
+
|
|
226
|
+
Args:
|
|
227
|
+
images (torch.Tensor): batch of images of shape [..., C, H, W]
|
|
228
|
+
|
|
229
|
+
crop_height (int): height of crop to take
|
|
230
|
+
|
|
231
|
+
crop_width (int): width of crop to take
|
|
232
|
+
|
|
233
|
+
num_crops (n): number of crops to sample
|
|
234
|
+
|
|
235
|
+
pos_enc (bool): if True, also add 2 channels to the outputs that gives a spatial
|
|
236
|
+
encoding of the original source pixel locations. This means that the
|
|
237
|
+
output crops will contain information about where in the source image
|
|
238
|
+
it was sampled from.
|
|
239
|
+
|
|
240
|
+
Returns:
|
|
241
|
+
crops (torch.Tensor): crops of shape (..., @num_crops, C, @crop_height, @crop_width)
|
|
242
|
+
if @pos_enc is False, otherwise (..., @num_crops, C + 2, @crop_height, @crop_width)
|
|
243
|
+
|
|
244
|
+
crop_inds (torch.Tensor): sampled crop indices of shape (..., N, 2)
|
|
245
|
+
"""
|
|
246
|
+
device = images.device
|
|
247
|
+
|
|
248
|
+
# maybe add 2 channels of spatial encoding to the source image
|
|
249
|
+
source_im = images
|
|
250
|
+
if pos_enc:
|
|
251
|
+
# spatial encoding [y, x] in [0, 1]
|
|
252
|
+
h, w = source_im.shape[-2:]
|
|
253
|
+
pos_y, pos_x = torch.meshgrid(torch.arange(h), torch.arange(w))
|
|
254
|
+
pos_y = pos_y.float().to(device) / float(h)
|
|
255
|
+
pos_x = pos_x.float().to(device) / float(w)
|
|
256
|
+
position_enc = torch.stack((pos_y, pos_x)) # shape [C, H, W]
|
|
257
|
+
|
|
258
|
+
# unsqueeze and expand to match leading dimensions -> shape [..., C, H, W]
|
|
259
|
+
leading_shape = source_im.shape[:-3]
|
|
260
|
+
position_enc = position_enc[(None,) * len(leading_shape)]
|
|
261
|
+
position_enc = position_enc.expand(*leading_shape, -1, -1, -1)
|
|
262
|
+
|
|
263
|
+
# concat across channel dimension with input
|
|
264
|
+
source_im = torch.cat((source_im, position_enc), dim=-3)
|
|
265
|
+
|
|
266
|
+
# make sure sample boundaries ensure crops are fully within the images
|
|
267
|
+
image_c, image_h, image_w = source_im.shape[-3:]
|
|
268
|
+
max_sample_h = image_h - crop_height
|
|
269
|
+
max_sample_w = image_w - crop_width
|
|
270
|
+
|
|
271
|
+
# Sample crop locations for all tensor dimensions up to the last 3, which are [C, H, W].
|
|
272
|
+
# Each gets @num_crops samples - typically this will just be the batch dimension (B), so
|
|
273
|
+
# we will sample [B, N] indices, but this supports having more than one leading dimension,
|
|
274
|
+
# or possibly no leading dimension.
|
|
275
|
+
#
|
|
276
|
+
# Trick: sample in [0, 1) with rand, then re-scale to [0, M) and convert to long to get sampled ints
|
|
277
|
+
crop_inds_h = (max_sample_h * torch.rand(*source_im.shape[:-3], num_crops).to(device)).long()
|
|
278
|
+
crop_inds_w = (max_sample_w * torch.rand(*source_im.shape[:-3], num_crops).to(device)).long()
|
|
279
|
+
crop_inds = torch.cat((crop_inds_h.unsqueeze(-1), crop_inds_w.unsqueeze(-1)), dim=-1) # shape [..., N, 2]
|
|
280
|
+
|
|
281
|
+
crops = crop_image_from_indices(
|
|
282
|
+
images=source_im,
|
|
283
|
+
crop_indices=crop_inds,
|
|
284
|
+
crop_height=crop_height,
|
|
285
|
+
crop_width=crop_width,
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
return crops, crop_inds
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
from typing import Dict, Tuple, Union
|
|
2
|
+
import copy
|
|
3
|
+
import torch
|
|
4
|
+
import torch.nn as nn
|
|
5
|
+
import torchvision
|
|
6
|
+
from diffusion_policy.model.vision.crop_randomizer import CropRandomizer
|
|
7
|
+
from diffusion_policy.model.common.module_attr_mixin import ModuleAttrMixin
|
|
8
|
+
from diffusion_policy.common.pytorch_util import dict_apply, replace_submodules
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class MultiImageObsEncoder(ModuleAttrMixin):
|
|
12
|
+
def __init__(self,
|
|
13
|
+
shape_meta: dict,
|
|
14
|
+
rgb_model: Union[nn.Module, Dict[str,nn.Module]],
|
|
15
|
+
resize_shape: Union[Tuple[int,int], Dict[str,tuple], None]=None,
|
|
16
|
+
crop_shape: Union[Tuple[int,int], Dict[str,tuple], None]=None,
|
|
17
|
+
random_crop: bool=True,
|
|
18
|
+
# replace BatchNorm with GroupNorm
|
|
19
|
+
use_group_norm: bool=False,
|
|
20
|
+
# use single rgb model for all rgb inputs
|
|
21
|
+
share_rgb_model: bool=False,
|
|
22
|
+
# renormalize rgb input with imagenet normalization
|
|
23
|
+
# assuming input in [0,1]
|
|
24
|
+
imagenet_norm: bool=False
|
|
25
|
+
):
|
|
26
|
+
"""
|
|
27
|
+
Assumes rgb input: B,C,H,W
|
|
28
|
+
Assumes low_dim input: B,D
|
|
29
|
+
"""
|
|
30
|
+
super().__init__()
|
|
31
|
+
|
|
32
|
+
rgb_keys = list()
|
|
33
|
+
low_dim_keys = list()
|
|
34
|
+
key_model_map = nn.ModuleDict()
|
|
35
|
+
key_transform_map = nn.ModuleDict()
|
|
36
|
+
key_shape_map = dict()
|
|
37
|
+
|
|
38
|
+
# handle sharing vision backbone
|
|
39
|
+
if share_rgb_model:
|
|
40
|
+
assert isinstance(rgb_model, nn.Module)
|
|
41
|
+
key_model_map['rgb'] = rgb_model
|
|
42
|
+
|
|
43
|
+
obs_shape_meta = shape_meta['obs']
|
|
44
|
+
for key, attr in obs_shape_meta.items():
|
|
45
|
+
shape = tuple(attr['shape'])
|
|
46
|
+
type = attr.get('type', 'low_dim')
|
|
47
|
+
key_shape_map[key] = shape
|
|
48
|
+
if type == 'rgb':
|
|
49
|
+
rgb_keys.append(key)
|
|
50
|
+
# configure model for this key
|
|
51
|
+
this_model = None
|
|
52
|
+
if not share_rgb_model:
|
|
53
|
+
if isinstance(rgb_model, dict):
|
|
54
|
+
# have provided model for each key
|
|
55
|
+
this_model = rgb_model[key]
|
|
56
|
+
else:
|
|
57
|
+
assert isinstance(rgb_model, nn.Module)
|
|
58
|
+
# have a copy of the rgb model
|
|
59
|
+
this_model = copy.deepcopy(rgb_model)
|
|
60
|
+
|
|
61
|
+
if this_model is not None:
|
|
62
|
+
if use_group_norm:
|
|
63
|
+
this_model = replace_submodules(
|
|
64
|
+
root_module=this_model,
|
|
65
|
+
predicate=lambda x: isinstance(x, nn.BatchNorm2d),
|
|
66
|
+
func=lambda x: nn.GroupNorm(
|
|
67
|
+
num_groups=x.num_features//16,
|
|
68
|
+
num_channels=x.num_features)
|
|
69
|
+
)
|
|
70
|
+
key_model_map[key] = this_model
|
|
71
|
+
|
|
72
|
+
# configure resize
|
|
73
|
+
input_shape = shape
|
|
74
|
+
this_resizer = nn.Identity()
|
|
75
|
+
if resize_shape is not None:
|
|
76
|
+
if isinstance(resize_shape, dict):
|
|
77
|
+
h, w = resize_shape[key]
|
|
78
|
+
else:
|
|
79
|
+
h, w = resize_shape
|
|
80
|
+
this_resizer = torchvision.transforms.Resize(
|
|
81
|
+
size=(h,w)
|
|
82
|
+
)
|
|
83
|
+
input_shape = (shape[0],h,w)
|
|
84
|
+
|
|
85
|
+
# configure randomizer
|
|
86
|
+
this_randomizer = nn.Identity()
|
|
87
|
+
if crop_shape is not None:
|
|
88
|
+
if isinstance(crop_shape, dict):
|
|
89
|
+
h, w = crop_shape[key]
|
|
90
|
+
else:
|
|
91
|
+
h, w = crop_shape
|
|
92
|
+
if random_crop:
|
|
93
|
+
this_randomizer = CropRandomizer(
|
|
94
|
+
input_shape=input_shape,
|
|
95
|
+
crop_height=h,
|
|
96
|
+
crop_width=w,
|
|
97
|
+
num_crops=1,
|
|
98
|
+
pos_enc=False
|
|
99
|
+
)
|
|
100
|
+
else:
|
|
101
|
+
this_normalizer = torchvision.transforms.CenterCrop(
|
|
102
|
+
size=(h,w)
|
|
103
|
+
)
|
|
104
|
+
# configure normalizer
|
|
105
|
+
this_normalizer = nn.Identity()
|
|
106
|
+
if imagenet_norm:
|
|
107
|
+
this_normalizer = torchvision.transforms.Normalize(
|
|
108
|
+
mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
|
|
109
|
+
|
|
110
|
+
this_transform = nn.Sequential(this_resizer, this_randomizer, this_normalizer)
|
|
111
|
+
key_transform_map[key] = this_transform
|
|
112
|
+
elif type == 'low_dim':
|
|
113
|
+
low_dim_keys.append(key)
|
|
114
|
+
else:
|
|
115
|
+
raise RuntimeError(f"Unsupported obs type: {type}")
|
|
116
|
+
rgb_keys = sorted(rgb_keys)
|
|
117
|
+
low_dim_keys = sorted(low_dim_keys)
|
|
118
|
+
|
|
119
|
+
self.shape_meta = shape_meta
|
|
120
|
+
self.key_model_map = key_model_map
|
|
121
|
+
self.key_transform_map = key_transform_map
|
|
122
|
+
self.share_rgb_model = share_rgb_model
|
|
123
|
+
self.rgb_keys = rgb_keys
|
|
124
|
+
self.low_dim_keys = low_dim_keys
|
|
125
|
+
self.key_shape_map = key_shape_map
|
|
126
|
+
|
|
127
|
+
def forward(self, obs_dict):
|
|
128
|
+
batch_size = None
|
|
129
|
+
features = list()
|
|
130
|
+
# process rgb input
|
|
131
|
+
if self.share_rgb_model:
|
|
132
|
+
# pass all rgb obs to rgb model
|
|
133
|
+
imgs = list()
|
|
134
|
+
for key in self.rgb_keys:
|
|
135
|
+
img = obs_dict[key]
|
|
136
|
+
if batch_size is None:
|
|
137
|
+
batch_size = img.shape[0]
|
|
138
|
+
else:
|
|
139
|
+
assert batch_size == img.shape[0]
|
|
140
|
+
assert img.shape[1:] == self.key_shape_map[key]
|
|
141
|
+
img = self.key_transform_map[key](img)
|
|
142
|
+
imgs.append(img)
|
|
143
|
+
# (N*B,C,H,W)
|
|
144
|
+
imgs = torch.cat(imgs, dim=0)
|
|
145
|
+
# (N*B,D)
|
|
146
|
+
feature = self.key_model_map['rgb'](imgs)
|
|
147
|
+
# (N,B,D)
|
|
148
|
+
feature = feature.reshape(-1,batch_size,*feature.shape[1:])
|
|
149
|
+
# (B,N,D)
|
|
150
|
+
feature = torch.moveaxis(feature,0,1)
|
|
151
|
+
# (B,N*D)
|
|
152
|
+
feature = feature.reshape(batch_size,-1)
|
|
153
|
+
features.append(feature)
|
|
154
|
+
else:
|
|
155
|
+
# run each rgb obs to independent models
|
|
156
|
+
for key in self.rgb_keys:
|
|
157
|
+
img = obs_dict[key]
|
|
158
|
+
if batch_size is None:
|
|
159
|
+
batch_size = img.shape[0]
|
|
160
|
+
else:
|
|
161
|
+
assert batch_size == img.shape[0]
|
|
162
|
+
assert img.shape[1:] == self.key_shape_map[key]
|
|
163
|
+
img = self.key_transform_map[key](img)
|
|
164
|
+
feature = self.key_model_map[key](img)
|
|
165
|
+
features.append(feature)
|
|
166
|
+
|
|
167
|
+
# process lowdim input
|
|
168
|
+
for key in self.low_dim_keys:
|
|
169
|
+
data = obs_dict[key]
|
|
170
|
+
if batch_size is None:
|
|
171
|
+
batch_size = data.shape[0]
|
|
172
|
+
else:
|
|
173
|
+
assert batch_size == data.shape[0]
|
|
174
|
+
assert data.shape[1:] == self.key_shape_map[key]
|
|
175
|
+
features.append(data)
|
|
176
|
+
|
|
177
|
+
# concatenate all features
|
|
178
|
+
result = torch.cat(features, dim=-1)
|
|
179
|
+
return result
|
|
180
|
+
|
|
181
|
+
@torch.no_grad()
|
|
182
|
+
def output_shape(self):
|
|
183
|
+
example_obs_dict = dict()
|
|
184
|
+
obs_shape_meta = self.shape_meta['obs']
|
|
185
|
+
batch_size = 1
|
|
186
|
+
for key, attr in obs_shape_meta.items():
|
|
187
|
+
shape = tuple(attr['shape'])
|
|
188
|
+
this_obs = torch.zeros(
|
|
189
|
+
(batch_size,) + shape,
|
|
190
|
+
dtype=self.dtype,
|
|
191
|
+
device=self.device)
|
|
192
|
+
example_obs_dict[key] = this_obs
|
|
193
|
+
example_output = self.forward(example_obs_dict)
|
|
194
|
+
output_shape = example_output.shape[1:]
|
|
195
|
+
return output_shape
|