solarian-diffusion 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. diffusion_policy/__init__.py +0 -0
  2. diffusion_policy/codecs/__init__.py +0 -0
  3. diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
  4. diffusion_policy/common/__init__.py +0 -0
  5. diffusion_policy/common/checkpoint_util.py +64 -0
  6. diffusion_policy/common/json_logger.py +117 -0
  7. diffusion_policy/common/normalize_util.py +251 -0
  8. diffusion_policy/common/pose_repr_util.py +112 -0
  9. diffusion_policy/common/pytorch_util.py +82 -0
  10. diffusion_policy/common/q_critic_util.py +135 -0
  11. diffusion_policy/common/replay_buffer.py +672 -0
  12. diffusion_policy/common/sampler.py +204 -0
  13. diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
  14. diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
  15. diffusion_policy/config/legacy/task/can_image.yaml +64 -0
  16. diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
  17. diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
  18. diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
  19. diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
  20. diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
  21. diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
  22. diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
  23. diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
  24. diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
  25. diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
  26. diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
  27. diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
  28. diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
  29. diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
  30. diffusion_policy/config/legacy/task/square_image.yaml +64 -0
  31. diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
  32. diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
  33. diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
  34. diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
  35. diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
  36. diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
  37. diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
  38. diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
  39. diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
  40. diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
  41. diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
  42. diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
  43. diffusion_policy/config/legacy/task/umi.yaml +45 -0
  44. diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
  45. diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
  46. diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
  47. diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
  48. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
  49. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
  50. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
  51. diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
  52. diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
  53. diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
  54. diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
  55. diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
  56. diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
  57. diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
  58. diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
  59. diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
  60. diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
  61. diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
  62. diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
  63. diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
  64. diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
  65. diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
  66. diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
  67. diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
  68. diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
  69. diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
  70. diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
  71. diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
  72. diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
  73. diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
  74. diffusion_policy/config/rl_annotate.yaml +265 -0
  75. diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
  76. diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
  77. diffusion_policy/config/rl_annotate_v2.yaml +141 -0
  78. diffusion_policy/config/rl_annotate_v3.yaml +165 -0
  79. diffusion_policy/config/task/lift.yaml +89 -0
  80. diffusion_policy/config/task/square.yaml +92 -0
  81. diffusion_policy/config/task/tool_hang.yaml +93 -0
  82. diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
  83. diffusion_policy/config/task/umi.yaml +105 -0
  84. diffusion_policy/config/task/umi_bimanual.yaml +135 -0
  85. diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
  86. diffusion_policy/config/task/umi_image.yaml +36 -0
  87. diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
  88. diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
  89. diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
  90. diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
  91. diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
  92. diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
  93. diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
  94. diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
  95. diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
  96. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
  97. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
  98. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
  99. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
  100. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
  101. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
  102. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
  103. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
  104. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
  105. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
  106. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
  107. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
  108. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
  109. diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
  110. diffusion_policy/config/train_iql_umi.yaml +186 -0
  111. diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
  112. diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
  113. diffusion_policy/dataset/__init__.py +0 -0
  114. diffusion_policy/dataset/base_dataset.py +76 -0
  115. diffusion_policy/dataset/umi_dataset.py +652 -0
  116. diffusion_policy/env_runner/__init__.py +0 -0
  117. diffusion_policy/env_runner/base_image_runner.py +9 -0
  118. diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
  119. diffusion_policy/model/__init__.py +0 -0
  120. diffusion_policy/model/common/__init__.py +0 -0
  121. diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
  122. diffusion_policy/model/common/lr_scheduler.py +46 -0
  123. diffusion_policy/model/common/module_attr_mixin.py +14 -0
  124. diffusion_policy/model/common/normalizer.py +353 -0
  125. diffusion_policy/model/common/tensor_util.py +960 -0
  126. diffusion_policy/model/diffusion/__init__.py +0 -0
  127. diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
  128. diffusion_policy/model/diffusion/conv1d_components.py +46 -0
  129. diffusion_policy/model/diffusion/ema_model.py +88 -0
  130. diffusion_policy/model/diffusion/mask_generator.py +221 -0
  131. diffusion_policy/model/diffusion/positional_embedding.py +17 -0
  132. diffusion_policy/model/vision/__init__.py +0 -0
  133. diffusion_policy/model/vision/crop_randomizer.py +288 -0
  134. diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
  135. diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
  136. diffusion_policy/policy/__init__.py +0 -0
  137. diffusion_policy/policy/base_image_policy.py +27 -0
  138. diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
  139. diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
  140. diffusion_policy/policy/parl_action_optimizer.py +675 -0
  141. diffusion_policy/workspace/__init__.py +0 -0
  142. diffusion_policy/workspace/base_workspace.py +146 -0
  143. diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
  144. solarian_diffusion/__init__.py +32 -0
  145. solarian_diffusion/config.py +31 -0
  146. solarian_diffusion/policy_server.py +316 -0
  147. solarian_diffusion/rtc.py +272 -0
  148. solarian_diffusion/service.py +16 -0
  149. solarian_diffusion/trainer.py +213 -0
  150. solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
  151. solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
  152. solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
  153. solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
  154. solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
  155. solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
  156. solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
  157. umi/__init__.py +0 -0
  158. umi/common/__init__.py +0 -0
  159. umi/common/pose_util.py +115 -0
@@ -0,0 +1,288 @@
1
+ import torch
2
+ import torch.nn as nn
3
+ import torchvision.transforms.functional as ttf
4
+ import diffusion_policy.model.common.tensor_util as tu
5
+
6
+ class CropRandomizer(nn.Module):
7
+ """
8
+ Randomly sample crops at input, and then average across crop features at output.
9
+ """
10
+ def __init__(
11
+ self,
12
+ input_shape,
13
+ crop_height,
14
+ crop_width,
15
+ num_crops=1,
16
+ pos_enc=False,
17
+ ):
18
+ """
19
+ Args:
20
+ input_shape (tuple, list): shape of input (not including batch dimension)
21
+ crop_height (int): crop height
22
+ crop_width (int): crop width
23
+ num_crops (int): number of random crops to take
24
+ pos_enc (bool): if True, add 2 channels to the output to encode the spatial
25
+ location of the cropped pixels in the source image
26
+ """
27
+ super().__init__()
28
+
29
+ assert len(input_shape) == 3 # (C, H, W)
30
+ assert crop_height < input_shape[1]
31
+ assert crop_width < input_shape[2]
32
+
33
+ self.input_shape = input_shape
34
+ self.crop_height = crop_height
35
+ self.crop_width = crop_width
36
+ self.num_crops = num_crops
37
+ self.pos_enc = pos_enc
38
+
39
+ def output_shape_in(self, input_shape=None):
40
+ """
41
+ Function to compute output shape from inputs to this module. Corresponds to
42
+ the @forward_in operation, where raw inputs (usually observation modalities)
43
+ are passed in.
44
+
45
+ Args:
46
+ input_shape (iterable of int): shape of input. Does not include batch dimension.
47
+ Some modules may not need this argument, if their output does not depend
48
+ on the size of the input, or if they assume fixed size input.
49
+
50
+ Returns:
51
+ out_shape ([int]): list of integers corresponding to output shape
52
+ """
53
+
54
+ # outputs are shape (C, CH, CW), or maybe C + 2 if using position encoding, because
55
+ # the number of crops are reshaped into the batch dimension, increasing the batch
56
+ # size from B to B * N
57
+ out_c = self.input_shape[0] + 2 if self.pos_enc else self.input_shape[0]
58
+ return [out_c, self.crop_height, self.crop_width]
59
+
60
+ def output_shape_out(self, input_shape=None):
61
+ """
62
+ Function to compute output shape from inputs to this module. Corresponds to
63
+ the @forward_out operation, where processed inputs (usually encoded observation
64
+ modalities) are passed in.
65
+
66
+ Args:
67
+ input_shape (iterable of int): shape of input. Does not include batch dimension.
68
+ Some modules may not need this argument, if their output does not depend
69
+ on the size of the input, or if they assume fixed size input.
70
+
71
+ Returns:
72
+ out_shape ([int]): list of integers corresponding to output shape
73
+ """
74
+
75
+ # since the forward_out operation splits [B * N, ...] -> [B, N, ...]
76
+ # and then pools to result in [B, ...], only the batch dimension changes,
77
+ # and so the other dimensions retain their shape.
78
+ return list(input_shape)
79
+
80
+ def forward_in(self, inputs):
81
+ """
82
+ Samples N random crops for each input in the batch, and then reshapes
83
+ inputs to [B * N, ...].
84
+ """
85
+ assert len(inputs.shape) >= 3 # must have at least (C, H, W) dimensions
86
+ if self.training:
87
+ # generate random crops
88
+ out, _ = sample_random_image_crops(
89
+ images=inputs,
90
+ crop_height=self.crop_height,
91
+ crop_width=self.crop_width,
92
+ num_crops=self.num_crops,
93
+ pos_enc=self.pos_enc,
94
+ )
95
+ # [B, N, ...] -> [B * N, ...]
96
+ return tu.join_dimensions(out, 0, 1)
97
+ else:
98
+ # take center crop during eval
99
+ out = ttf.center_crop(img=inputs, output_size=(
100
+ self.crop_height, self.crop_width))
101
+ if self.num_crops > 1:
102
+ B,C,H,W = out.shape
103
+ out = out.unsqueeze(1).expand(B,self.num_crops,C,H,W).reshape(-1,C,H,W)
104
+ # [B * N, ...]
105
+ return out
106
+
107
+ def forward_out(self, inputs):
108
+ """
109
+ Splits the outputs from shape [B * N, ...] -> [B, N, ...] and then average across N
110
+ to result in shape [B, ...] to make sure the network output is consistent with
111
+ what would have happened if there were no randomization.
112
+ """
113
+ if self.num_crops <= 1:
114
+ return inputs
115
+ else:
116
+ batch_size = (inputs.shape[0] // self.num_crops)
117
+ out = tu.reshape_dimensions(inputs, begin_axis=0, end_axis=0,
118
+ target_dims=(batch_size, self.num_crops))
119
+ return out.mean(dim=1)
120
+
121
+ def forward(self, inputs):
122
+ return self.forward_in(inputs)
123
+
124
+ def __repr__(self):
125
+ """Pretty print network."""
126
+ header = '{}'.format(str(self.__class__.__name__))
127
+ msg = header + "(input_shape={}, crop_size=[{}, {}], num_crops={})".format(
128
+ self.input_shape, self.crop_height, self.crop_width, self.num_crops)
129
+ return msg
130
+
131
+
132
+ def crop_image_from_indices(images, crop_indices, crop_height, crop_width):
133
+ """
134
+ Crops images at the locations specified by @crop_indices. Crops will be
135
+ taken across all channels.
136
+
137
+ Args:
138
+ images (torch.Tensor): batch of images of shape [..., C, H, W]
139
+
140
+ crop_indices (torch.Tensor): batch of indices of shape [..., N, 2] where
141
+ N is the number of crops to take per image and each entry corresponds
142
+ to the pixel height and width of where to take the crop. Note that
143
+ the indices can also be of shape [..., 2] if only 1 crop should
144
+ be taken per image. Leading dimensions must be consistent with
145
+ @images argument. Each index specifies the top left of the crop.
146
+ Values must be in range [0, H - CH - 1] x [0, W - CW - 1] where
147
+ H and W are the height and width of @images and CH and CW are
148
+ @crop_height and @crop_width.
149
+
150
+ crop_height (int): height of crop to take
151
+
152
+ crop_width (int): width of crop to take
153
+
154
+ Returns:
155
+ crops (torch.Tesnor): cropped images of shape [..., C, @crop_height, @crop_width]
156
+ """
157
+
158
+ # make sure length of input shapes is consistent
159
+ assert crop_indices.shape[-1] == 2
160
+ ndim_im_shape = len(images.shape)
161
+ ndim_indices_shape = len(crop_indices.shape)
162
+ assert (ndim_im_shape == ndim_indices_shape + 1) or (ndim_im_shape == ndim_indices_shape + 2)
163
+
164
+ # maybe pad so that @crop_indices is shape [..., N, 2]
165
+ is_padded = False
166
+ if ndim_im_shape == ndim_indices_shape + 2:
167
+ crop_indices = crop_indices.unsqueeze(-2)
168
+ is_padded = True
169
+
170
+ # make sure leading dimensions between images and indices are consistent
171
+ assert images.shape[:-3] == crop_indices.shape[:-2]
172
+
173
+ device = images.device
174
+ image_c, image_h, image_w = images.shape[-3:]
175
+ num_crops = crop_indices.shape[-2]
176
+
177
+ # make sure @crop_indices are in valid range
178
+ assert (crop_indices[..., 0] >= 0).all().item()
179
+ assert (crop_indices[..., 0] < (image_h - crop_height)).all().item()
180
+ assert (crop_indices[..., 1] >= 0).all().item()
181
+ assert (crop_indices[..., 1] < (image_w - crop_width)).all().item()
182
+
183
+ # convert each crop index (ch, cw) into a list of pixel indices that correspond to the entire window.
184
+
185
+ # 2D index array with columns [0, 1, ..., CH - 1] and shape [CH, CW]
186
+ crop_ind_grid_h = torch.arange(crop_height).to(device)
187
+ crop_ind_grid_h = tu.unsqueeze_expand_at(crop_ind_grid_h, size=crop_width, dim=-1)
188
+ # 2D index array with rows [0, 1, ..., CW - 1] and shape [CH, CW]
189
+ crop_ind_grid_w = torch.arange(crop_width).to(device)
190
+ crop_ind_grid_w = tu.unsqueeze_expand_at(crop_ind_grid_w, size=crop_height, dim=0)
191
+ # combine into shape [CH, CW, 2]
192
+ crop_in_grid = torch.cat((crop_ind_grid_h.unsqueeze(-1), crop_ind_grid_w.unsqueeze(-1)), dim=-1)
193
+
194
+ # Add above grid with the offset index of each sampled crop to get 2d indices for each crop.
195
+ # After broadcasting, this will be shape [..., N, CH, CW, 2] and each crop has a [CH, CW, 2]
196
+ # shape array that tells us which pixels from the corresponding source image to grab.
197
+ grid_reshape = [1] * len(crop_indices.shape[:-1]) + [crop_height, crop_width, 2]
198
+ all_crop_inds = crop_indices.unsqueeze(-2).unsqueeze(-2) + crop_in_grid.reshape(grid_reshape)
199
+
200
+ # For using @torch.gather, convert to flat indices from 2D indices, and also
201
+ # repeat across the channel dimension. To get flat index of each pixel to grab for
202
+ # each sampled crop, we just use the mapping: ind = h_ind * @image_w + w_ind
203
+ all_crop_inds = all_crop_inds[..., 0] * image_w + all_crop_inds[..., 1] # shape [..., N, CH, CW]
204
+ all_crop_inds = tu.unsqueeze_expand_at(all_crop_inds, size=image_c, dim=-3) # shape [..., N, C, CH, CW]
205
+ all_crop_inds = tu.flatten(all_crop_inds, begin_axis=-2) # shape [..., N, C, CH * CW]
206
+
207
+ # Repeat and flatten the source images -> [..., N, C, H * W] and then use gather to index with crop pixel inds
208
+ images_to_crop = tu.unsqueeze_expand_at(images, size=num_crops, dim=-4)
209
+ images_to_crop = tu.flatten(images_to_crop, begin_axis=-2)
210
+ crops = torch.gather(images_to_crop, dim=-1, index=all_crop_inds)
211
+ # [..., N, C, CH * CW] -> [..., N, C, CH, CW]
212
+ reshape_axis = len(crops.shape) - 1
213
+ crops = tu.reshape_dimensions(crops, begin_axis=reshape_axis, end_axis=reshape_axis,
214
+ target_dims=(crop_height, crop_width))
215
+
216
+ if is_padded:
217
+ # undo padding -> [..., C, CH, CW]
218
+ crops = crops.squeeze(-4)
219
+ return crops
220
+
221
+ def sample_random_image_crops(images, crop_height, crop_width, num_crops, pos_enc=False):
222
+ """
223
+ For each image, randomly sample @num_crops crops of size (@crop_height, @crop_width), from
224
+ @images.
225
+
226
+ Args:
227
+ images (torch.Tensor): batch of images of shape [..., C, H, W]
228
+
229
+ crop_height (int): height of crop to take
230
+
231
+ crop_width (int): width of crop to take
232
+
233
+ num_crops (n): number of crops to sample
234
+
235
+ pos_enc (bool): if True, also add 2 channels to the outputs that gives a spatial
236
+ encoding of the original source pixel locations. This means that the
237
+ output crops will contain information about where in the source image
238
+ it was sampled from.
239
+
240
+ Returns:
241
+ crops (torch.Tensor): crops of shape (..., @num_crops, C, @crop_height, @crop_width)
242
+ if @pos_enc is False, otherwise (..., @num_crops, C + 2, @crop_height, @crop_width)
243
+
244
+ crop_inds (torch.Tensor): sampled crop indices of shape (..., N, 2)
245
+ """
246
+ device = images.device
247
+
248
+ # maybe add 2 channels of spatial encoding to the source image
249
+ source_im = images
250
+ if pos_enc:
251
+ # spatial encoding [y, x] in [0, 1]
252
+ h, w = source_im.shape[-2:]
253
+ pos_y, pos_x = torch.meshgrid(torch.arange(h), torch.arange(w))
254
+ pos_y = pos_y.float().to(device) / float(h)
255
+ pos_x = pos_x.float().to(device) / float(w)
256
+ position_enc = torch.stack((pos_y, pos_x)) # shape [C, H, W]
257
+
258
+ # unsqueeze and expand to match leading dimensions -> shape [..., C, H, W]
259
+ leading_shape = source_im.shape[:-3]
260
+ position_enc = position_enc[(None,) * len(leading_shape)]
261
+ position_enc = position_enc.expand(*leading_shape, -1, -1, -1)
262
+
263
+ # concat across channel dimension with input
264
+ source_im = torch.cat((source_im, position_enc), dim=-3)
265
+
266
+ # make sure sample boundaries ensure crops are fully within the images
267
+ image_c, image_h, image_w = source_im.shape[-3:]
268
+ max_sample_h = image_h - crop_height
269
+ max_sample_w = image_w - crop_width
270
+
271
+ # Sample crop locations for all tensor dimensions up to the last 3, which are [C, H, W].
272
+ # Each gets @num_crops samples - typically this will just be the batch dimension (B), so
273
+ # we will sample [B, N] indices, but this supports having more than one leading dimension,
274
+ # or possibly no leading dimension.
275
+ #
276
+ # Trick: sample in [0, 1) with rand, then re-scale to [0, M) and convert to long to get sampled ints
277
+ crop_inds_h = (max_sample_h * torch.rand(*source_im.shape[:-3], num_crops).to(device)).long()
278
+ crop_inds_w = (max_sample_w * torch.rand(*source_im.shape[:-3], num_crops).to(device)).long()
279
+ crop_inds = torch.cat((crop_inds_h.unsqueeze(-1), crop_inds_w.unsqueeze(-1)), dim=-1) # shape [..., N, 2]
280
+
281
+ crops = crop_image_from_indices(
282
+ images=source_im,
283
+ crop_indices=crop_inds,
284
+ crop_height=crop_height,
285
+ crop_width=crop_width,
286
+ )
287
+
288
+ return crops, crop_inds
@@ -0,0 +1,195 @@
1
+ from typing import Dict, Tuple, Union
2
+ import copy
3
+ import torch
4
+ import torch.nn as nn
5
+ import torchvision
6
+ from diffusion_policy.model.vision.crop_randomizer import CropRandomizer
7
+ from diffusion_policy.model.common.module_attr_mixin import ModuleAttrMixin
8
+ from diffusion_policy.common.pytorch_util import dict_apply, replace_submodules
9
+
10
+
11
+ class MultiImageObsEncoder(ModuleAttrMixin):
12
+ def __init__(self,
13
+ shape_meta: dict,
14
+ rgb_model: Union[nn.Module, Dict[str,nn.Module]],
15
+ resize_shape: Union[Tuple[int,int], Dict[str,tuple], None]=None,
16
+ crop_shape: Union[Tuple[int,int], Dict[str,tuple], None]=None,
17
+ random_crop: bool=True,
18
+ # replace BatchNorm with GroupNorm
19
+ use_group_norm: bool=False,
20
+ # use single rgb model for all rgb inputs
21
+ share_rgb_model: bool=False,
22
+ # renormalize rgb input with imagenet normalization
23
+ # assuming input in [0,1]
24
+ imagenet_norm: bool=False
25
+ ):
26
+ """
27
+ Assumes rgb input: B,C,H,W
28
+ Assumes low_dim input: B,D
29
+ """
30
+ super().__init__()
31
+
32
+ rgb_keys = list()
33
+ low_dim_keys = list()
34
+ key_model_map = nn.ModuleDict()
35
+ key_transform_map = nn.ModuleDict()
36
+ key_shape_map = dict()
37
+
38
+ # handle sharing vision backbone
39
+ if share_rgb_model:
40
+ assert isinstance(rgb_model, nn.Module)
41
+ key_model_map['rgb'] = rgb_model
42
+
43
+ obs_shape_meta = shape_meta['obs']
44
+ for key, attr in obs_shape_meta.items():
45
+ shape = tuple(attr['shape'])
46
+ type = attr.get('type', 'low_dim')
47
+ key_shape_map[key] = shape
48
+ if type == 'rgb':
49
+ rgb_keys.append(key)
50
+ # configure model for this key
51
+ this_model = None
52
+ if not share_rgb_model:
53
+ if isinstance(rgb_model, dict):
54
+ # have provided model for each key
55
+ this_model = rgb_model[key]
56
+ else:
57
+ assert isinstance(rgb_model, nn.Module)
58
+ # have a copy of the rgb model
59
+ this_model = copy.deepcopy(rgb_model)
60
+
61
+ if this_model is not None:
62
+ if use_group_norm:
63
+ this_model = replace_submodules(
64
+ root_module=this_model,
65
+ predicate=lambda x: isinstance(x, nn.BatchNorm2d),
66
+ func=lambda x: nn.GroupNorm(
67
+ num_groups=x.num_features//16,
68
+ num_channels=x.num_features)
69
+ )
70
+ key_model_map[key] = this_model
71
+
72
+ # configure resize
73
+ input_shape = shape
74
+ this_resizer = nn.Identity()
75
+ if resize_shape is not None:
76
+ if isinstance(resize_shape, dict):
77
+ h, w = resize_shape[key]
78
+ else:
79
+ h, w = resize_shape
80
+ this_resizer = torchvision.transforms.Resize(
81
+ size=(h,w)
82
+ )
83
+ input_shape = (shape[0],h,w)
84
+
85
+ # configure randomizer
86
+ this_randomizer = nn.Identity()
87
+ if crop_shape is not None:
88
+ if isinstance(crop_shape, dict):
89
+ h, w = crop_shape[key]
90
+ else:
91
+ h, w = crop_shape
92
+ if random_crop:
93
+ this_randomizer = CropRandomizer(
94
+ input_shape=input_shape,
95
+ crop_height=h,
96
+ crop_width=w,
97
+ num_crops=1,
98
+ pos_enc=False
99
+ )
100
+ else:
101
+ this_normalizer = torchvision.transforms.CenterCrop(
102
+ size=(h,w)
103
+ )
104
+ # configure normalizer
105
+ this_normalizer = nn.Identity()
106
+ if imagenet_norm:
107
+ this_normalizer = torchvision.transforms.Normalize(
108
+ mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
109
+
110
+ this_transform = nn.Sequential(this_resizer, this_randomizer, this_normalizer)
111
+ key_transform_map[key] = this_transform
112
+ elif type == 'low_dim':
113
+ low_dim_keys.append(key)
114
+ else:
115
+ raise RuntimeError(f"Unsupported obs type: {type}")
116
+ rgb_keys = sorted(rgb_keys)
117
+ low_dim_keys = sorted(low_dim_keys)
118
+
119
+ self.shape_meta = shape_meta
120
+ self.key_model_map = key_model_map
121
+ self.key_transform_map = key_transform_map
122
+ self.share_rgb_model = share_rgb_model
123
+ self.rgb_keys = rgb_keys
124
+ self.low_dim_keys = low_dim_keys
125
+ self.key_shape_map = key_shape_map
126
+
127
+ def forward(self, obs_dict):
128
+ batch_size = None
129
+ features = list()
130
+ # process rgb input
131
+ if self.share_rgb_model:
132
+ # pass all rgb obs to rgb model
133
+ imgs = list()
134
+ for key in self.rgb_keys:
135
+ img = obs_dict[key]
136
+ if batch_size is None:
137
+ batch_size = img.shape[0]
138
+ else:
139
+ assert batch_size == img.shape[0]
140
+ assert img.shape[1:] == self.key_shape_map[key]
141
+ img = self.key_transform_map[key](img)
142
+ imgs.append(img)
143
+ # (N*B,C,H,W)
144
+ imgs = torch.cat(imgs, dim=0)
145
+ # (N*B,D)
146
+ feature = self.key_model_map['rgb'](imgs)
147
+ # (N,B,D)
148
+ feature = feature.reshape(-1,batch_size,*feature.shape[1:])
149
+ # (B,N,D)
150
+ feature = torch.moveaxis(feature,0,1)
151
+ # (B,N*D)
152
+ feature = feature.reshape(batch_size,-1)
153
+ features.append(feature)
154
+ else:
155
+ # run each rgb obs to independent models
156
+ for key in self.rgb_keys:
157
+ img = obs_dict[key]
158
+ if batch_size is None:
159
+ batch_size = img.shape[0]
160
+ else:
161
+ assert batch_size == img.shape[0]
162
+ assert img.shape[1:] == self.key_shape_map[key]
163
+ img = self.key_transform_map[key](img)
164
+ feature = self.key_model_map[key](img)
165
+ features.append(feature)
166
+
167
+ # process lowdim input
168
+ for key in self.low_dim_keys:
169
+ data = obs_dict[key]
170
+ if batch_size is None:
171
+ batch_size = data.shape[0]
172
+ else:
173
+ assert batch_size == data.shape[0]
174
+ assert data.shape[1:] == self.key_shape_map[key]
175
+ features.append(data)
176
+
177
+ # concatenate all features
178
+ result = torch.cat(features, dim=-1)
179
+ return result
180
+
181
+ @torch.no_grad()
182
+ def output_shape(self):
183
+ example_obs_dict = dict()
184
+ obs_shape_meta = self.shape_meta['obs']
185
+ batch_size = 1
186
+ for key, attr in obs_shape_meta.items():
187
+ shape = tuple(attr['shape'])
188
+ this_obs = torch.zeros(
189
+ (batch_size,) + shape,
190
+ dtype=self.dtype,
191
+ device=self.device)
192
+ example_obs_dict[key] = this_obs
193
+ example_output = self.forward(example_obs_dict)
194
+ output_shape = example_output.shape[1:]
195
+ return output_shape