solarian-diffusion 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. diffusion_policy/__init__.py +0 -0
  2. diffusion_policy/codecs/__init__.py +0 -0
  3. diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
  4. diffusion_policy/common/__init__.py +0 -0
  5. diffusion_policy/common/checkpoint_util.py +64 -0
  6. diffusion_policy/common/json_logger.py +117 -0
  7. diffusion_policy/common/normalize_util.py +251 -0
  8. diffusion_policy/common/pose_repr_util.py +112 -0
  9. diffusion_policy/common/pytorch_util.py +82 -0
  10. diffusion_policy/common/q_critic_util.py +135 -0
  11. diffusion_policy/common/replay_buffer.py +672 -0
  12. diffusion_policy/common/sampler.py +204 -0
  13. diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
  14. diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
  15. diffusion_policy/config/legacy/task/can_image.yaml +64 -0
  16. diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
  17. diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
  18. diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
  19. diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
  20. diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
  21. diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
  22. diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
  23. diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
  24. diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
  25. diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
  26. diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
  27. diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
  28. diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
  29. diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
  30. diffusion_policy/config/legacy/task/square_image.yaml +64 -0
  31. diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
  32. diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
  33. diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
  34. diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
  35. diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
  36. diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
  37. diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
  38. diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
  39. diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
  40. diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
  41. diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
  42. diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
  43. diffusion_policy/config/legacy/task/umi.yaml +45 -0
  44. diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
  45. diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
  46. diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
  47. diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
  48. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
  49. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
  50. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
  51. diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
  52. diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
  53. diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
  54. diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
  55. diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
  56. diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
  57. diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
  58. diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
  59. diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
  60. diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
  61. diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
  62. diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
  63. diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
  64. diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
  65. diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
  66. diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
  67. diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
  68. diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
  69. diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
  70. diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
  71. diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
  72. diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
  73. diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
  74. diffusion_policy/config/rl_annotate.yaml +265 -0
  75. diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
  76. diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
  77. diffusion_policy/config/rl_annotate_v2.yaml +141 -0
  78. diffusion_policy/config/rl_annotate_v3.yaml +165 -0
  79. diffusion_policy/config/task/lift.yaml +89 -0
  80. diffusion_policy/config/task/square.yaml +92 -0
  81. diffusion_policy/config/task/tool_hang.yaml +93 -0
  82. diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
  83. diffusion_policy/config/task/umi.yaml +105 -0
  84. diffusion_policy/config/task/umi_bimanual.yaml +135 -0
  85. diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
  86. diffusion_policy/config/task/umi_image.yaml +36 -0
  87. diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
  88. diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
  89. diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
  90. diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
  91. diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
  92. diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
  93. diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
  94. diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
  95. diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
  96. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
  97. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
  98. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
  99. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
  100. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
  101. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
  102. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
  103. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
  104. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
  105. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
  106. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
  107. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
  108. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
  109. diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
  110. diffusion_policy/config/train_iql_umi.yaml +186 -0
  111. diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
  112. diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
  113. diffusion_policy/dataset/__init__.py +0 -0
  114. diffusion_policy/dataset/base_dataset.py +76 -0
  115. diffusion_policy/dataset/umi_dataset.py +652 -0
  116. diffusion_policy/env_runner/__init__.py +0 -0
  117. diffusion_policy/env_runner/base_image_runner.py +9 -0
  118. diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
  119. diffusion_policy/model/__init__.py +0 -0
  120. diffusion_policy/model/common/__init__.py +0 -0
  121. diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
  122. diffusion_policy/model/common/lr_scheduler.py +46 -0
  123. diffusion_policy/model/common/module_attr_mixin.py +14 -0
  124. diffusion_policy/model/common/normalizer.py +353 -0
  125. diffusion_policy/model/common/tensor_util.py +960 -0
  126. diffusion_policy/model/diffusion/__init__.py +0 -0
  127. diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
  128. diffusion_policy/model/diffusion/conv1d_components.py +46 -0
  129. diffusion_policy/model/diffusion/ema_model.py +88 -0
  130. diffusion_policy/model/diffusion/mask_generator.py +221 -0
  131. diffusion_policy/model/diffusion/positional_embedding.py +17 -0
  132. diffusion_policy/model/vision/__init__.py +0 -0
  133. diffusion_policy/model/vision/crop_randomizer.py +288 -0
  134. diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
  135. diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
  136. diffusion_policy/policy/__init__.py +0 -0
  137. diffusion_policy/policy/base_image_policy.py +27 -0
  138. diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
  139. diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
  140. diffusion_policy/policy/parl_action_optimizer.py +675 -0
  141. diffusion_policy/workspace/__init__.py +0 -0
  142. diffusion_policy/workspace/base_workspace.py +146 -0
  143. diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
  144. solarian_diffusion/__init__.py +32 -0
  145. solarian_diffusion/config.py +31 -0
  146. solarian_diffusion/policy_server.py +316 -0
  147. solarian_diffusion/rtc.py +272 -0
  148. solarian_diffusion/service.py +16 -0
  149. solarian_diffusion/trainer.py +213 -0
  150. solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
  151. solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
  152. solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
  153. solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
  154. solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
  155. solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
  156. solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
  157. umi/__init__.py +0 -0
  158. umi/common/__init__.py +0 -0
  159. umi/common/pose_util.py +115 -0
@@ -0,0 +1,525 @@
1
+ import copy
2
+
3
+ import timm
4
+ import math
5
+ import torch
6
+ import torch.nn as nn
7
+ import torch.nn.functional as F
8
+ import torchvision
9
+ import logging
10
+
11
+ from diffusion_policy.model.common.module_attr_mixin import ModuleAttrMixin
12
+
13
+ from diffusion_policy.common.pytorch_util import replace_submodules
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+
18
+ def _expand_vit_patch_embed_to_4ch(model: nn.Module) -> None:
19
+ """Replace ``model.patch_embed.proj`` with a 4-input-channel conv.
20
+
21
+ The first three input-channel slices are copied verbatim from the
22
+ pretrained conv so the RGB path is bit-identical at initialization;
23
+ the new 4th input-channel slice is zero-initialized so the modified
24
+ network's output equals the original network's output on RGB-only
25
+ inputs at step 0. The optimizer then grows the 4th-channel weights
26
+ only when the mask carries gradient-reducing information.
27
+
28
+ Implements the "zero-init weight expansion" / identity-preserving-at-init
29
+ recipe used to bolt new modalities onto pretrained backbones.
30
+ """
31
+ if not (hasattr(model, "patch_embed") and hasattr(model.patch_embed, "proj")):
32
+ raise RuntimeError(
33
+ "bbox_mode=mask_channel requires a ViT-style encoder with "
34
+ "`patch_embed.proj` (a Conv2d). Got a model of type "
35
+ f"{type(model).__name__} without that structure."
36
+ )
37
+ old_proj: nn.Conv2d = model.patch_embed.proj
38
+ if not isinstance(old_proj, nn.Conv2d):
39
+ raise RuntimeError(
40
+ f"patch_embed.proj is not a Conv2d (got {type(old_proj).__name__})."
41
+ )
42
+ if old_proj.in_channels != 3:
43
+ raise RuntimeError(
44
+ "Expected pretrained patch_embed.proj.in_channels == 3, got "
45
+ f"{old_proj.in_channels}. Refusing to overwrite — the encoder "
46
+ "may already have been surgically modified."
47
+ )
48
+
49
+ new_proj = nn.Conv2d(
50
+ in_channels=4,
51
+ out_channels=old_proj.out_channels,
52
+ kernel_size=old_proj.kernel_size,
53
+ stride=old_proj.stride,
54
+ padding=old_proj.padding,
55
+ dilation=old_proj.dilation,
56
+ groups=old_proj.groups,
57
+ bias=(old_proj.bias is not None),
58
+ )
59
+ with torch.no_grad():
60
+ new_proj.weight[:, :3].copy_(old_proj.weight)
61
+ new_proj.weight[:, 3:].zero_()
62
+ if old_proj.bias is not None:
63
+ new_proj.bias.copy_(old_proj.bias)
64
+ model.patch_embed.proj = new_proj
65
+
66
+
67
+ def rasterize_bbox_to_mask(bbox: torch.Tensor, H: int, W: int) -> torch.Tensor:
68
+ """Rasterize pixel-space bboxes to a binary mask.
69
+
70
+ Args:
71
+ bbox: tensor of shape ``(..., 4)`` with ``[x0, y0, x1, y1]`` in
72
+ pixel coordinates of the ``H x W`` image.
73
+ H: target mask height in pixels.
74
+ W: target mask width in pixels.
75
+
76
+ Returns:
77
+ Float tensor of shape ``(..., 1, H, W)`` with values in ``{0, 1}``.
78
+ Pixels strictly inside ``[x0, x1) x [y0, y1)`` are 1.
79
+ """
80
+ leading_shape = bbox.shape[:-1]
81
+ flat = bbox.reshape(-1, 4)
82
+ N = flat.shape[0]
83
+
84
+ device = flat.device
85
+ # Use float32 for the comparison grid regardless of bbox dtype so we
86
+ # don't lose precision when the encoder is running under bf16/fp16.
87
+ xs = torch.arange(W, device=device, dtype=torch.float32).view(1, 1, 1, W)
88
+ ys = torch.arange(H, device=device, dtype=torch.float32).view(1, 1, H, 1)
89
+
90
+ flat_f = flat.to(torch.float32)
91
+ x0 = flat_f[:, 0].view(N, 1, 1, 1)
92
+ y0 = flat_f[:, 1].view(N, 1, 1, 1)
93
+ x1 = flat_f[:, 2].view(N, 1, 1, 1)
94
+ y1 = flat_f[:, 3].view(N, 1, 1, 1)
95
+
96
+ in_x = (xs >= x0) & (xs < x1)
97
+ in_y = (ys >= y0) & (ys < y1)
98
+ mask = (in_x & in_y).to(bbox.dtype)
99
+ return mask.reshape(*leading_shape, 1, H, W)
100
+
101
+
102
+ class MaskAwareImageTransform(nn.Module):
103
+ """Apply geometric transforms to all channels, color transforms only to RGB.
104
+
105
+ Used when the input image has been augmented with extra non-RGB channels
106
+ (e.g. a binary bbox mask). RandomCrop / Resize keep the mask aligned with
107
+ the image; ColorJitter / Normalize would corrupt the mask if applied to
108
+ all channels, so they are restricted to the first three (RGB) channels.
109
+ """
110
+
111
+ def __init__(self, geometric: list, rgb_only: list, n_rgb: int = 3):
112
+ super().__init__()
113
+ self.geometric = nn.Sequential(*geometric) if geometric else nn.Identity()
114
+ self.rgb_only = nn.Sequential(*rgb_only) if rgb_only else nn.Identity()
115
+ self.n_rgb = n_rgb
116
+
117
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
118
+ x = self.geometric(x)
119
+ if x.shape[1] <= self.n_rgb:
120
+ return self.rgb_only(x)
121
+ rgb = self.rgb_only(x[:, : self.n_rgb])
122
+ rest = x[:, self.n_rgb:]
123
+ return torch.cat([rgb, rest], dim=1)
124
+
125
+ class AttentionPool2d(nn.Module):
126
+ def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None):
127
+ super().__init__()
128
+ self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5)
129
+ self.k_proj = nn.Linear(embed_dim, embed_dim)
130
+ self.q_proj = nn.Linear(embed_dim, embed_dim)
131
+ self.v_proj = nn.Linear(embed_dim, embed_dim)
132
+ self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim)
133
+ self.num_heads = num_heads
134
+
135
+ def forward(self, x):
136
+ x = x.flatten(start_dim=2).permute(2, 0, 1) # NCHW -> (HW)NC
137
+ x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC
138
+ x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC
139
+ x, _ = F.multi_head_attention_forward(
140
+ query=x[:1], key=x, value=x,
141
+ embed_dim_to_check=x.shape[-1],
142
+ num_heads=self.num_heads,
143
+ q_proj_weight=self.q_proj.weight,
144
+ k_proj_weight=self.k_proj.weight,
145
+ v_proj_weight=self.v_proj.weight,
146
+ in_proj_weight=None,
147
+ in_proj_bias=torch.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]),
148
+ bias_k=None,
149
+ bias_v=None,
150
+ add_zero_attn=False,
151
+ dropout_p=0,
152
+ out_proj_weight=self.c_proj.weight,
153
+ out_proj_bias=self.c_proj.bias,
154
+ use_separate_proj_weight=True,
155
+ training=self.training,
156
+ need_weights=False
157
+ )
158
+ return x.squeeze(0)
159
+
160
+
161
+ class TimmObsEncoder(ModuleAttrMixin):
162
+ def __init__(self,
163
+ shape_meta: dict,
164
+ model_name: str,
165
+ pretrained: bool,
166
+ frozen: bool,
167
+ global_pool: str,
168
+ transforms: list,
169
+ # replace BatchNorm with GroupNorm
170
+ use_group_norm: bool=False,
171
+ # use single rgb model for all rgb inputs
172
+ share_rgb_model: bool=False,
173
+ # renormalize rgb input with imagenet normalization
174
+ # assuming input in [0,1]
175
+ imagenet_norm: bool=False,
176
+ feature_aggregation: str='spatial_embedding',
177
+ downsample_ratio: int=32,
178
+ position_encording: str='learnable',
179
+
180
+ ):
181
+ """
182
+ Assumes rgb input: B,T,C,H,W
183
+ Assumes low_dim input: B,T,D
184
+ """
185
+ super().__init__()
186
+
187
+ rgb_keys = list()
188
+ low_dim_keys = list()
189
+ key_model_map = nn.ModuleDict()
190
+ key_transform_map = nn.ModuleDict()
191
+ key_shape_map = dict()
192
+
193
+ assert global_pool == ''
194
+ model = timm.create_model(
195
+ model_name=model_name,
196
+ pretrained=pretrained,
197
+ global_pool=global_pool, # '' means no pooling
198
+ num_classes=0 # remove classification layer
199
+ )
200
+
201
+ if frozen:
202
+ assert pretrained
203
+ for param in model.parameters():
204
+ param.requires_grad = False
205
+
206
+ feature_dim = None
207
+ if model_name.startswith('resnet'):
208
+ # the last layer is nn.Identity() because num_classes is 0
209
+ # second last layer is AdaptivePool2d, which is also identity because global_pool is empty
210
+ if downsample_ratio == 32:
211
+ modules = list(model.children())[:-2]
212
+ model = torch.nn.Sequential(*modules)
213
+ feature_dim = 512
214
+ elif downsample_ratio == 16:
215
+ modules = list(model.children())[:-3]
216
+ model = torch.nn.Sequential(*modules)
217
+ feature_dim = 256
218
+ else:
219
+ raise NotImplementedError(f"Unsupported downsample_ratio: {downsample_ratio}")
220
+ elif model_name.startswith('convnext'):
221
+ # the last layer is nn.Identity() because num_classes is 0
222
+ # second last layer is AdaptivePool2d, which is also identity because global_pool is empty
223
+ if downsample_ratio == 32:
224
+ modules = list(model.children())[:-2]
225
+ model = torch.nn.Sequential(*modules)
226
+ feature_dim = 1024
227
+ else:
228
+ raise NotImplementedError(f"Unsupported downsample_ratio: {downsample_ratio}")
229
+
230
+ if use_group_norm and not pretrained:
231
+ model = replace_submodules(
232
+ root_module=model,
233
+ predicate=lambda x: isinstance(x, nn.BatchNorm2d),
234
+ func=lambda x: nn.GroupNorm(
235
+ num_groups=(x.num_features // 16) if (x.num_features % 16 == 0) else (x.num_features // 8),
236
+ num_channels=x.num_features)
237
+ )
238
+
239
+ image_shape = None
240
+ obs_shape_meta = shape_meta['obs']
241
+ for key, attr in obs_shape_meta.items():
242
+ shape = tuple(attr['shape'])
243
+ type = attr.get('type', 'low_dim')
244
+ if type == 'rgb':
245
+ assert image_shape is None or image_shape == shape[1:]
246
+ image_shape = shape[1:]
247
+
248
+ # ---- bbox -> mask-channel routing -----------------------------------
249
+ # Scan low_dim obs entries for `bbox_mode == 'mask_channel'`. Each such
250
+ # entry is rasterized into a binary mask and concatenated as a 4th
251
+ # channel onto a target RGB key inside forward(). The bbox itself is
252
+ # removed from the low-dim concat path so it doesn't get fed twice.
253
+ # `bbox_mask_map`: rgb_key -> bbox_key (the bbox routed onto that rgb).
254
+ bbox_mask_map = dict()
255
+ bbox_mask_horizon = dict() # rgb_key -> expected matching horizon
256
+ for key, attr in obs_shape_meta.items():
257
+ if attr.get('type', 'low_dim') != 'low_dim':
258
+ continue
259
+ if attr.get('bbox_mode', 'low_dim') != 'mask_channel':
260
+ continue
261
+ shape = tuple(attr['shape'])
262
+ if shape != (4,):
263
+ raise RuntimeError(
264
+ f"bbox_mode=mask_channel requires shape [4] (x0,y0,x1,y1) "
265
+ f"for key '{key}', got {shape}.")
266
+ target_rgb_key = attr.get('target_rgb_key')
267
+ if target_rgb_key is None:
268
+ rgb_candidates = [
269
+ k for k, a in obs_shape_meta.items()
270
+ if a.get('type', 'low_dim') == 'rgb']
271
+ if len(rgb_candidates) != 1:
272
+ raise RuntimeError(
273
+ f"bbox key '{key}' with bbox_mode=mask_channel did not "
274
+ f"specify `target_rgb_key` and there are "
275
+ f"{len(rgb_candidates)} candidate rgb keys; please set "
276
+ f"`target_rgb_key` explicitly in shape_meta.")
277
+ target_rgb_key = rgb_candidates[0]
278
+ if target_rgb_key not in obs_shape_meta or \
279
+ obs_shape_meta[target_rgb_key].get('type') != 'rgb':
280
+ raise RuntimeError(
281
+ f"bbox key '{key}' targets rgb_key='{target_rgb_key}' "
282
+ f"which is not an rgb obs entry.")
283
+ if target_rgb_key in bbox_mask_map:
284
+ raise RuntimeError(
285
+ f"Multiple bbox keys are routed to rgb_key="
286
+ f"'{target_rgb_key}'. Only one bbox mask channel per "
287
+ f"image is supported.")
288
+ rgb_horizon = int(obs_shape_meta[target_rgb_key]['horizon'])
289
+ bbox_horizon = int(attr['horizon'])
290
+ if bbox_horizon != rgb_horizon:
291
+ raise RuntimeError(
292
+ f"bbox '{key}' (horizon={bbox_horizon}) and target image "
293
+ f"'{target_rgb_key}' (horizon={rgb_horizon}) must share "
294
+ f"the same horizon when bbox_mode=mask_channel.")
295
+ bbox_mask_map[target_rgb_key] = key
296
+ bbox_mask_horizon[target_rgb_key] = rgb_horizon
297
+
298
+ if transforms is not None and not isinstance(transforms[0], torch.nn.Module):
299
+ assert transforms[0].type == 'RandomCrop'
300
+ ratio = transforms[0].ratio
301
+ transforms = [
302
+ torchvision.transforms.RandomCrop(size=int(image_shape[0] * ratio)),
303
+ torchvision.transforms.Resize(size=image_shape[0], antialias=True)
304
+ ] + transforms[1:]
305
+ # The two leading transforms (RandomCrop, Resize) are geometric and
306
+ # are safe on arbitrary channel counts; everything after them is RGB-
307
+ # only (ColorJitter uses HSV, ImageNet Normalize assumes 3 channels).
308
+ geometric_transforms = transforms[:2] if transforms is not None else []
309
+ color_transforms = transforms[2:] if transforms is not None else []
310
+ rgb_only_transform = (
311
+ nn.Identity() if transforms is None
312
+ else torch.nn.Sequential(*transforms)
313
+ )
314
+
315
+ for key, attr in obs_shape_meta.items():
316
+ shape = tuple(attr['shape'])
317
+ type = attr.get('type', 'low_dim')
318
+ key_shape_map[key] = shape
319
+ if type == 'rgb':
320
+ rgb_keys.append(key)
321
+
322
+ this_model = model if share_rgb_model else copy.deepcopy(model)
323
+ if key in bbox_mask_map:
324
+ # Expand the patch embedding to accept the mask channel.
325
+ # Zero-init the new slice so the model output is identical
326
+ # to the pretrained model at step 0 (see header docstring).
327
+ if share_rgb_model:
328
+ raise RuntimeError(
329
+ "bbox_mode=mask_channel is incompatible with "
330
+ "share_rgb_model=True (the patch_embed surgery "
331
+ "would be applied multiple times to the same "
332
+ "tensor). Disable share_rgb_model or unset the "
333
+ "mask routing for this key.")
334
+ _expand_vit_patch_embed_to_4ch(this_model)
335
+ this_transform = MaskAwareImageTransform(
336
+ geometric=geometric_transforms,
337
+ rgb_only=color_transforms,
338
+ )
339
+ else:
340
+ this_transform = rgb_only_transform
341
+ key_model_map[key] = this_model
342
+ key_transform_map[key] = this_transform
343
+ elif type == 'low_dim':
344
+ if attr.get('ignore_by_policy', False):
345
+ continue
346
+ if attr.get('bbox_mode', 'low_dim') == 'mask_channel':
347
+ # consumed via the rgb path, not as a flat feature.
348
+ continue
349
+ low_dim_keys.append(key)
350
+ else:
351
+ raise RuntimeError(f"Unsupported obs type: {type}")
352
+
353
+ feature_map_shape = [x // downsample_ratio for x in image_shape]
354
+
355
+ rgb_keys = sorted(rgb_keys)
356
+ low_dim_keys = sorted(low_dim_keys)
357
+ print('rgb keys: ', rgb_keys)
358
+ print('low_dim_keys keys:', low_dim_keys)
359
+
360
+ self.model_name = model_name
361
+ self.shape_meta = shape_meta
362
+ self.key_model_map = key_model_map
363
+ self.key_transform_map = key_transform_map
364
+ self.share_rgb_model = share_rgb_model
365
+ self.rgb_keys = rgb_keys
366
+ self.low_dim_keys = low_dim_keys
367
+ self.key_shape_map = key_shape_map
368
+ self.feature_aggregation = feature_aggregation
369
+ # rgb_key -> bbox_key. Empty dict means no mask routing is active and
370
+ # forward() behaves exactly as before. Persisted on the encoder so
371
+ # downstream code (eval, loggers, debug tools) can introspect the
372
+ # bbox integration mode without re-parsing shape_meta.
373
+ self.bbox_mask_map = bbox_mask_map
374
+ if model_name.startswith('vit'):
375
+ # assert self.feature_aggregation is None # vit uses the CLS token
376
+ if self.feature_aggregation == 'all_tokens':
377
+ # Use all tokens from ViT
378
+ pass
379
+ elif self.feature_aggregation is not None:
380
+ logger.warn(f'vit will use the CLS token. feature_aggregation ({self.feature_aggregation}) is ignored!')
381
+ self.feature_aggregation = None
382
+
383
+ if self.feature_aggregation == 'soft_attention':
384
+ self.attention = nn.Sequential(
385
+ nn.Linear(feature_dim, 1, bias=False),
386
+ nn.Softmax(dim=1)
387
+ )
388
+ elif self.feature_aggregation == 'spatial_embedding':
389
+ self.spatial_embedding = torch.nn.Parameter(torch.randn(feature_map_shape[0] * feature_map_shape[1], feature_dim))
390
+ elif self.feature_aggregation == 'transformer':
391
+ if position_encording == 'learnable':
392
+ self.position_embedding = torch.nn.Parameter(torch.randn(feature_map_shape[0] * feature_map_shape[1] + 1, feature_dim))
393
+ elif position_encording == 'sinusoidal':
394
+ num_features = feature_map_shape[0] * feature_map_shape[1] + 1
395
+ self.position_embedding = torch.zeros(num_features, feature_dim)
396
+ position = torch.arange(0, num_features, dtype=torch.float).unsqueeze(1)
397
+ div_term = torch.exp(torch.arange(0, feature_dim, 2).float() * (-math.log(2 * num_features) / feature_dim))
398
+ self.position_embedding[:, 0::2] = torch.sin(position * div_term)
399
+ self.position_embedding[:, 1::2] = torch.cos(position * div_term)
400
+ self.aggregation_transformer = nn.TransformerEncoder(
401
+ encoder_layer=nn.TransformerEncoderLayer(d_model=feature_dim, nhead=4),
402
+ num_layers=4)
403
+ elif self.feature_aggregation == 'attention_pool_2d':
404
+ self.attention_pool_2d = AttentionPool2d(
405
+ spacial_dim=feature_map_shape[0],
406
+ embed_dim=feature_dim,
407
+ num_heads=feature_dim // 64,
408
+ output_dim=feature_dim
409
+ )
410
+ logger.info(
411
+ "number of parameters: %e", sum(p.numel() for p in self.parameters())
412
+ )
413
+
414
+ def aggregate_feature(self, feature):
415
+ if self.model_name.startswith('vit'):
416
+ assert self.feature_aggregation is None # vit uses the CLS token
417
+ return feature[:, 0, :]
418
+
419
+ # resnet
420
+ assert len(feature.shape) == 4
421
+ if self.feature_aggregation == 'attention_pool_2d':
422
+ return self.attention_pool_2d(feature)
423
+
424
+ feature = torch.flatten(feature, start_dim=-2) # B, 512, 7*7
425
+ feature = torch.transpose(feature, 1, 2) # B, 7*7, 512
426
+
427
+ if self.feature_aggregation == 'avg':
428
+ return torch.mean(feature, dim=[1])
429
+ elif self.feature_aggregation == 'max':
430
+ return torch.amax(feature, dim=[1])
431
+ elif self.feature_aggregation == 'soft_attention':
432
+ weight = self.attention(feature)
433
+ return torch.sum(feature * weight, dim=1)
434
+ elif self.feature_aggregation == 'spatial_embedding':
435
+ return torch.mean(feature * self.spatial_embedding, dim=1)
436
+ elif self.feature_aggregation == 'transformer':
437
+ zero_feature = torch.zeros(feature.shape[0], 1, feature.shape[-1], device=feature.device)
438
+ if self.position_embedding.device != feature.device:
439
+ self.position_embedding = self.position_embedding.to(feature.device)
440
+ feature_with_pos_embedding = torch.concat([zero_feature, feature], dim=1) + self.position_embedding
441
+ feature_output = self.aggregation_transformer(feature_with_pos_embedding)
442
+ return feature_output[:, 0]
443
+ else:
444
+ assert self.feature_aggregation is None
445
+ return feature
446
+
447
+ def forward(self, obs_dict):
448
+ features = list()
449
+ batch_size = next(iter(obs_dict.values())).shape[0]
450
+
451
+ # process rgb input
452
+ for key in self.rgb_keys:
453
+ img = obs_dict[key]
454
+ B, T = img.shape[:2]
455
+ assert B == batch_size
456
+ assert img.shape[2:] == self.key_shape_map[key]
457
+
458
+ if key in self.bbox_mask_map:
459
+ # Rasterize the routed bbox into a (B, T, 1, H, W) binary
460
+ # mask and concat as a 4th channel. Geometric transforms in
461
+ # MaskAwareImageTransform then crop/resize image+mask
462
+ # together, keeping the bbox geometry consistent with the
463
+ # image content the ViT sees.
464
+ bbox_key = self.bbox_mask_map[key]
465
+ bbox = obs_dict[bbox_key]
466
+ assert bbox.shape[0] == batch_size, (
467
+ f"bbox '{bbox_key}' batch={bbox.shape[0]} mismatches "
468
+ f"image '{key}' batch={batch_size}")
469
+ assert bbox.shape[1] == T, (
470
+ f"bbox '{bbox_key}' horizon={bbox.shape[1]} mismatches "
471
+ f"image '{key}' horizon={T}")
472
+ assert bbox.shape[2:] == (4,), (
473
+ f"bbox '{bbox_key}' expected shape [4], got "
474
+ f"{tuple(bbox.shape[2:])}")
475
+ _, _, _, H, W = img.shape
476
+ mask = rasterize_bbox_to_mask(bbox, H=H, W=W).to(img.dtype)
477
+ img = torch.cat([img, mask], dim=2)
478
+
479
+ img = img.reshape(B*T, *img.shape[2:])
480
+ img = self.key_transform_map[key](img)
481
+ raw_feature = self.key_model_map[key](img)
482
+ feature = self.aggregate_feature(raw_feature)
483
+ assert len(feature.shape) == 2 and feature.shape[0] == B * T
484
+ features.append(feature.reshape(B, -1))
485
+
486
+ # process lowdim input
487
+ for key in self.low_dim_keys:
488
+ data = obs_dict[key]
489
+ B, T = data.shape[:2]
490
+ assert B == batch_size
491
+ assert data.shape[2:] == self.key_shape_map[key]
492
+ features.append(data.reshape(B, -1))
493
+
494
+ # concatenate all features
495
+ result = torch.cat(features, dim=-1)
496
+
497
+ return result
498
+
499
+
500
+ @torch.no_grad()
501
+ def output_shape(self):
502
+ example_obs_dict = dict()
503
+ obs_shape_meta = self.shape_meta['obs']
504
+ for key, attr in obs_shape_meta.items():
505
+ shape = tuple(attr['shape'])
506
+ this_obs = torch.zeros(
507
+ (1, attr['horizon']) + shape,
508
+ dtype=self.dtype,
509
+ device=self.device)
510
+ example_obs_dict[key] = this_obs
511
+ example_output = self.forward(example_obs_dict)
512
+ assert len(example_output.shape) == 2
513
+ assert example_output.shape[0] == 1
514
+
515
+ return example_output.shape
516
+
517
+
518
+ if __name__=='__main__':
519
+ timm_obs_encoder = TimmObsEncoder(
520
+ shape_meta=None,
521
+ model_name='resnet18.a1_in1k',
522
+ pretrained=False,
523
+ global_pool='',
524
+ transforms=None
525
+ )
File without changes
@@ -0,0 +1,27 @@
1
+ from typing import Dict
2
+ import torch
3
+ import torch.nn as nn
4
+ from diffusion_policy.model.common.module_attr_mixin import ModuleAttrMixin
5
+ from diffusion_policy.model.common.normalizer import LinearNormalizer
6
+
7
+ class BaseImagePolicy(ModuleAttrMixin):
8
+ # init accepts keyword argument shape_meta, see config/task/*_image.yaml
9
+
10
+ def predict_action(self, obs_dict: Dict[str, torch.Tensor], fixed_action_prefix: torch.Tensor=None) -> Dict[str, torch.Tensor]:
11
+ """
12
+ obs_dict:
13
+ str: B,To,*
14
+ fixed_action_prefix:
15
+ B, Tp, Da
16
+ return: B,Ta,Da
17
+ """
18
+ raise NotImplementedError()
19
+
20
+ # reset state for stateful policies
21
+ def reset(self):
22
+ pass
23
+
24
+ # ========== training ===========
25
+ # no standard training interface except setting normalizer
26
+ def set_normalizer(self, normalizer: LinearNormalizer):
27
+ raise NotImplementedError()