solarian-diffusion 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. diffusion_policy/__init__.py +0 -0
  2. diffusion_policy/codecs/__init__.py +0 -0
  3. diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
  4. diffusion_policy/common/__init__.py +0 -0
  5. diffusion_policy/common/checkpoint_util.py +64 -0
  6. diffusion_policy/common/json_logger.py +117 -0
  7. diffusion_policy/common/normalize_util.py +251 -0
  8. diffusion_policy/common/pose_repr_util.py +112 -0
  9. diffusion_policy/common/pytorch_util.py +82 -0
  10. diffusion_policy/common/q_critic_util.py +135 -0
  11. diffusion_policy/common/replay_buffer.py +672 -0
  12. diffusion_policy/common/sampler.py +204 -0
  13. diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
  14. diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
  15. diffusion_policy/config/legacy/task/can_image.yaml +64 -0
  16. diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
  17. diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
  18. diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
  19. diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
  20. diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
  21. diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
  22. diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
  23. diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
  24. diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
  25. diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
  26. diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
  27. diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
  28. diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
  29. diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
  30. diffusion_policy/config/legacy/task/square_image.yaml +64 -0
  31. diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
  32. diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
  33. diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
  34. diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
  35. diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
  36. diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
  37. diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
  38. diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
  39. diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
  40. diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
  41. diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
  42. diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
  43. diffusion_policy/config/legacy/task/umi.yaml +45 -0
  44. diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
  45. diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
  46. diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
  47. diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
  48. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
  49. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
  50. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
  51. diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
  52. diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
  53. diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
  54. diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
  55. diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
  56. diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
  57. diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
  58. diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
  59. diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
  60. diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
  61. diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
  62. diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
  63. diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
  64. diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
  65. diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
  66. diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
  67. diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
  68. diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
  69. diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
  70. diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
  71. diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
  72. diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
  73. diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
  74. diffusion_policy/config/rl_annotate.yaml +265 -0
  75. diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
  76. diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
  77. diffusion_policy/config/rl_annotate_v2.yaml +141 -0
  78. diffusion_policy/config/rl_annotate_v3.yaml +165 -0
  79. diffusion_policy/config/task/lift.yaml +89 -0
  80. diffusion_policy/config/task/square.yaml +92 -0
  81. diffusion_policy/config/task/tool_hang.yaml +93 -0
  82. diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
  83. diffusion_policy/config/task/umi.yaml +105 -0
  84. diffusion_policy/config/task/umi_bimanual.yaml +135 -0
  85. diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
  86. diffusion_policy/config/task/umi_image.yaml +36 -0
  87. diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
  88. diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
  89. diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
  90. diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
  91. diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
  92. diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
  93. diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
  94. diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
  95. diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
  96. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
  97. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
  98. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
  99. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
  100. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
  101. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
  102. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
  103. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
  104. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
  105. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
  106. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
  107. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
  108. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
  109. diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
  110. diffusion_policy/config/train_iql_umi.yaml +186 -0
  111. diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
  112. diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
  113. diffusion_policy/dataset/__init__.py +0 -0
  114. diffusion_policy/dataset/base_dataset.py +76 -0
  115. diffusion_policy/dataset/umi_dataset.py +652 -0
  116. diffusion_policy/env_runner/__init__.py +0 -0
  117. diffusion_policy/env_runner/base_image_runner.py +9 -0
  118. diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
  119. diffusion_policy/model/__init__.py +0 -0
  120. diffusion_policy/model/common/__init__.py +0 -0
  121. diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
  122. diffusion_policy/model/common/lr_scheduler.py +46 -0
  123. diffusion_policy/model/common/module_attr_mixin.py +14 -0
  124. diffusion_policy/model/common/normalizer.py +353 -0
  125. diffusion_policy/model/common/tensor_util.py +960 -0
  126. diffusion_policy/model/diffusion/__init__.py +0 -0
  127. diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
  128. diffusion_policy/model/diffusion/conv1d_components.py +46 -0
  129. diffusion_policy/model/diffusion/ema_model.py +88 -0
  130. diffusion_policy/model/diffusion/mask_generator.py +221 -0
  131. diffusion_policy/model/diffusion/positional_embedding.py +17 -0
  132. diffusion_policy/model/vision/__init__.py +0 -0
  133. diffusion_policy/model/vision/crop_randomizer.py +288 -0
  134. diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
  135. diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
  136. diffusion_policy/policy/__init__.py +0 -0
  137. diffusion_policy/policy/base_image_policy.py +27 -0
  138. diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
  139. diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
  140. diffusion_policy/policy/parl_action_optimizer.py +675 -0
  141. diffusion_policy/workspace/__init__.py +0 -0
  142. diffusion_policy/workspace/base_workspace.py +146 -0
  143. diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
  144. solarian_diffusion/__init__.py +32 -0
  145. solarian_diffusion/config.py +31 -0
  146. solarian_diffusion/policy_server.py +316 -0
  147. solarian_diffusion/rtc.py +272 -0
  148. solarian_diffusion/service.py +16 -0
  149. solarian_diffusion/trainer.py +213 -0
  150. solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
  151. solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
  152. solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
  153. solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
  154. solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
  155. solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
  156. solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
  157. umi/__init__.py +0 -0
  158. umi/common/__init__.py +0 -0
  159. umi/common/pose_util.py +115 -0
@@ -0,0 +1,607 @@
1
+ import copy
2
+ from typing import Dict, Optional
3
+ import torch
4
+ import torch.nn as nn
5
+ import torch.nn.functional as F
6
+ import numpy as np
7
+ from einops import rearrange, reduce
8
+ from diffusers.schedulers.scheduling_ddpm import DDPMScheduler
9
+
10
+ from diffusion_policy.model.common.normalizer import LinearNormalizer
11
+ from diffusion_policy.policy.base_image_policy import BaseImagePolicy
12
+ from diffusion_policy.model.diffusion.conditional_unet1d import ConditionalUnet1D
13
+ from diffusion_policy.model.diffusion.mask_generator import LowdimMaskGenerator
14
+ from diffusion_policy.model.vision.timm_obs_encoder import TimmObsEncoder
15
+ from diffusion_policy.common.pytorch_util import dict_apply
16
+
17
+
18
+ class DiffusionUnetTimmPolicy(BaseImagePolicy):
19
+ def __init__(self,
20
+ shape_meta: dict,
21
+ noise_scheduler: DDPMScheduler,
22
+ obs_encoder: TimmObsEncoder,
23
+ num_inference_steps=None,
24
+ obs_as_global_cond=True,
25
+ diffusion_step_embed_dim=256,
26
+ down_dims=(256,512,1024),
27
+ kernel_size=5,
28
+ n_groups=8,
29
+ cond_predict_scale=True,
30
+ input_pertub=0.1,
31
+ inpaint_fixed_action_prefix=False,
32
+ train_diffusion_n_samples=1,
33
+ # ============== AWR finetuning knobs ==============
34
+ # Advantage-Weighted Regression: weight each sample's
35
+ # denoising MSE by ``exp(advantage / awr_beta)``. Disabled
36
+ # (legacy BC) when awr_beta <= 0 OR the batch lacks
37
+ # ``advantage`` / ``has_advantage`` keys (which UmiDataset
38
+ # adds when rl/q + rl/v are present in the source zarr).
39
+ #
40
+ # awr_beta: temperature on the advantage. Lower beta
41
+ # sharpens the weighting (more like Q-max but
42
+ # unstable); higher beta softens it toward BC.
43
+ # A sensible default is roughly ``std(A)`` on
44
+ # your dataset (your IQL ckpt name encodes
45
+ # ``val_adv_std``, so use that as a starting
46
+ # point). 0.0 disables AWR entirely.
47
+ # awr_w_max: per-sample weight clip BEFORE batch norm.
48
+ # Caps the influence of a single outlier with
49
+ # very large advantage. Typical 10..100.
50
+ # awr_lambda: blend between BC weight (1.0) and AWR weight
51
+ # ``w``. 1.0 is pure AWR; 0.0 collapses back
52
+ # to BC; values in (0, 1) act as a "safety
53
+ # belt" toward the BC checkpoint's behavior.
54
+ # awr_normalize_weights: if True, divide weights by their
55
+ # batch mean before applying. Makes the
56
+ # effective LR roughly invariant to beta and
57
+ # to which subset of the data has high
58
+ # advantage. Strongly recommended.
59
+ # awr_use_ema_weights: deprecated knob slot kept here as a
60
+ # future hook (e.g. for a slow-moving running
61
+ # mean of the batch weight to stabilize early
62
+ # training); currently unused.
63
+ awr_beta=0.0,
64
+ awr_w_max=20.0,
65
+ awr_lambda=1.0,
66
+ awr_normalize_weights=True,
67
+ # ============== BC-anchored AWR knobs ==============
68
+ # The plain AWR loss above can overfit to high-advantage
69
+ # outliers and drift the policy off the BC manifold,
70
+ # producing twitchy / unstable rollouts. The three extra
71
+ # knobs below let us anchor the fine-tune back toward BC:
72
+ #
73
+ # awr_w_min: lower clamp on per-sample weights
74
+ # AFTER the log-cap. Prevents the
75
+ # optimizer from completely ignoring
76
+ # low-advantage but on-distribution
77
+ # samples (the prior says "still do
78
+ # what BC said to do"). Typical
79
+ # 0.3..0.5; 0 reproduces legacy AWR.
80
+ # awr_bc_lambda: coefficient on the UNWEIGHTED BC
81
+ # loss term (plain denoising MSE,
82
+ # SAME pred/target as the AWR term so
83
+ # there's no extra forward). Pulls
84
+ # the policy back toward "imitate
85
+ # every demo equally". Typical
86
+ # 0.5..5.0; 0 disables the term.
87
+ # awr_anchor_lambda: coefficient on the FROZEN-REFERENCE
88
+ # anchor: MSE between the current
89
+ # policy's eps-prediction and a
90
+ # frozen copy of the BC policy's
91
+ # eps-prediction at the same
92
+ # (x_t, t, obs). Acts as a soft KL
93
+ # regularizer toward the BC policy
94
+ # in eps-space. Requires the
95
+ # workspace to call
96
+ # ``init_frozen_ref()`` once after
97
+ # loading ``init_from_ckpt``.
98
+ # Typical 0.1..1.0; 0 disables.
99
+ awr_w_min=0.0,
100
+ awr_bc_lambda=0.0,
101
+ awr_anchor_lambda=0.0,
102
+ # parameters passed to step
103
+ **kwargs
104
+ ):
105
+ super().__init__()
106
+
107
+ # parse shapes
108
+ action_shape = shape_meta['action']['shape']
109
+ assert len(action_shape) == 1
110
+ action_dim = action_shape[0]
111
+ action_horizon = shape_meta['action']['horizon']
112
+ # get feature dim
113
+ obs_feature_dim = np.prod(obs_encoder.output_shape())
114
+
115
+
116
+ # create diffusion model
117
+ assert obs_as_global_cond
118
+ input_dim = action_dim
119
+ global_cond_dim = obs_feature_dim
120
+
121
+ model = ConditionalUnet1D(
122
+ input_dim=input_dim,
123
+ local_cond_dim=None,
124
+ global_cond_dim=global_cond_dim,
125
+ diffusion_step_embed_dim=diffusion_step_embed_dim,
126
+ down_dims=down_dims,
127
+ kernel_size=kernel_size,
128
+ n_groups=n_groups,
129
+ cond_predict_scale=cond_predict_scale
130
+ )
131
+
132
+ self.obs_encoder = obs_encoder
133
+ self.model = model
134
+ self.noise_scheduler = noise_scheduler
135
+ self.normalizer = LinearNormalizer()
136
+ self.obs_feature_dim = obs_feature_dim
137
+ self.action_dim = action_dim
138
+ self.action_horizon = action_horizon # used for training
139
+ self.obs_as_global_cond = obs_as_global_cond
140
+ self.input_pertub = input_pertub
141
+ self.inpaint_fixed_action_prefix = inpaint_fixed_action_prefix
142
+ self.train_diffusion_n_samples = int(train_diffusion_n_samples)
143
+ self.kwargs = kwargs
144
+
145
+ if num_inference_steps is None:
146
+ num_inference_steps = noise_scheduler.config.num_train_timesteps
147
+ self.num_inference_steps = num_inference_steps
148
+
149
+ # AWR finetuning state. ``last_awr_info`` carries per-batch
150
+ # diagnostic scalars (weight stats, fraction of zero-advantage
151
+ # samples, etc.) that the workspace pulls into wandb. Set to
152
+ # None when AWR is disabled or the batch had no advantage info.
153
+ self.awr_beta = float(awr_beta)
154
+ self.awr_w_max = float(awr_w_max)
155
+ self.awr_lambda = float(awr_lambda)
156
+ self.awr_normalize_weights = bool(awr_normalize_weights)
157
+ self.awr_w_min = float(awr_w_min)
158
+ self.awr_bc_lambda = float(awr_bc_lambda)
159
+ self.awr_anchor_lambda = float(awr_anchor_lambda)
160
+ # Frozen reference U-Net for the anchor term. Materialized
161
+ # lazily by `init_frozen_ref()` (called by the workspace AFTER
162
+ # `init_from_ckpt` has loaded the BC weights into self.model).
163
+ # Excluded from `state_dict()` so AWR checkpoints don't double
164
+ # in size (the ref is re-built from `init_from_ckpt` on the
165
+ # next finetune; for inference loads it's never needed).
166
+ self.frozen_ref_model: Optional[nn.Module] = None
167
+ self.last_awr_info: Dict[str, float] = {}
168
+
169
+ def init_frozen_ref(self) -> None:
170
+ """Snapshot the current self.model as a frozen anchor.
171
+
172
+ Called by the workspace right after ``init_from_ckpt`` has
173
+ loaded the BC weights, so the frozen reference IS the BC
174
+ policy at finetune-step-0. No-op when the anchor term is
175
+ disabled (``awr_anchor_lambda <= 0``) so the legacy AWR run
176
+ doesn't pay the extra ~600 MB of GPU memory for a copy of
177
+ the U-Net it'll never use.
178
+ """
179
+ if self.awr_anchor_lambda <= 0.0:
180
+ return
181
+ if self.frozen_ref_model is not None:
182
+ return # idempotent
183
+ # Deep-copy ensures parameter tensors don't share storage with
184
+ # self.model -- otherwise an in-place op on self.model (or
185
+ # accelerator-wrapping) could mutate the anchor.
186
+ ref = copy.deepcopy(self.model).eval()
187
+ for p in ref.parameters():
188
+ p.requires_grad_(False)
189
+ # We register as an attribute (not a submodule) so it is NOT
190
+ # part of `self.parameters()` and therefore not picked up by
191
+ # the optimizer / DDP / EMA. The eval-mode forward is invoked
192
+ # explicitly from compute_loss.
193
+ object.__setattr__(self, 'frozen_ref_model', ref)
194
+
195
+ def state_dict(self, *args, **kwargs):
196
+ """Strip the frozen reference from saved checkpoints.
197
+
198
+ Including it would double the on-disk size of every AWR
199
+ checkpoint for zero benefit (the reference is rebuilt from
200
+ ``init_from_ckpt`` on the next finetune run, and inference
201
+ loads don't need it at all).
202
+ """
203
+ sd = super().state_dict(*args, **kwargs)
204
+ keys_to_drop = [k for k in sd.keys() if k.startswith('frozen_ref_model.')]
205
+ for k in keys_to_drop:
206
+ del sd[k]
207
+ return sd
208
+
209
+ def load_state_dict(self, state_dict, strict: bool = True, *args, **kwargs):
210
+ """Tolerate the absence of frozen_ref_model.* keys on load.
211
+
212
+ AWR checkpoints (which we save via the overridden
213
+ ``state_dict`` above) never contain those keys, but a
214
+ downstream ``init_frozen_ref()`` re-materializes the buffer
215
+ from current weights so nothing is lost.
216
+ """
217
+ if strict:
218
+ ref_keys = [k for k in self.state_dict().keys()
219
+ if k.startswith('frozen_ref_model.')]
220
+ if ref_keys and not any(
221
+ k.startswith('frozen_ref_model.') for k in state_dict.keys()):
222
+ # Demote strict load for this specific case.
223
+ strict = False
224
+ return super().load_state_dict(state_dict, strict=strict, *args, **kwargs)
225
+
226
+ # ========= inference ============
227
+ def conditional_sample(self,
228
+ condition_data,
229
+ condition_mask,
230
+ local_cond=None,
231
+ global_cond=None,
232
+ generator=None,
233
+ # keyword arguments to scheduler.step
234
+ **kwargs
235
+ ):
236
+ model = self.model
237
+ scheduler = self.noise_scheduler
238
+
239
+ trajectory = torch.randn(
240
+ size=condition_data.shape,
241
+ dtype=condition_data.dtype,
242
+ device=condition_data.device,
243
+ generator=generator)
244
+
245
+ # set step values
246
+ scheduler.set_timesteps(self.num_inference_steps)
247
+
248
+ for t in scheduler.timesteps:
249
+ # 1. apply conditioning
250
+ trajectory[condition_mask] = condition_data[condition_mask]
251
+
252
+ # 2. predict model output
253
+ model_output = model(trajectory, t,
254
+ local_cond=local_cond, global_cond=global_cond)
255
+
256
+ # 3. compute previous image: x_t -> x_t-1
257
+ trajectory = scheduler.step(
258
+ model_output, t, trajectory,
259
+ generator=generator,
260
+ **kwargs
261
+ ).prev_sample
262
+
263
+ # finally make sure conditioning is enforced
264
+ trajectory[condition_mask] = condition_data[condition_mask]
265
+
266
+ return trajectory
267
+
268
+
269
+ def predict_action(self, obs_dict: Dict[str, torch.Tensor], fixed_action_prefix: torch.Tensor=None) -> Dict[str, torch.Tensor]:
270
+ """
271
+ obs_dict: must include "obs" key
272
+ fixed_action_prefix: unnormalized action prefix
273
+ result: must include "action" key
274
+ """
275
+ assert 'past_action' not in obs_dict # not implemented yet
276
+ # normalize input
277
+ nobs = self.normalizer.normalize(obs_dict)
278
+ B = next(iter(nobs.values())).shape[0]
279
+
280
+ # condition through global feature
281
+ global_cond = self.obs_encoder(nobs)
282
+
283
+ # empty data for action
284
+ cond_data = torch.zeros(size=(B, self.action_horizon, self.action_dim), device=self.device, dtype=self.dtype)
285
+ cond_mask = torch.zeros_like(cond_data, dtype=torch.bool)
286
+
287
+ if fixed_action_prefix is not None and self.inpaint_fixed_action_prefix:
288
+ n_fixed_steps = fixed_action_prefix.shape[1]
289
+ cond_data[:, :n_fixed_steps] = fixed_action_prefix
290
+ cond_mask[:, :n_fixed_steps] = True
291
+ cond_data = self.normalizer['action'].normalize(cond_data)
292
+
293
+
294
+ # run sampling
295
+ nsample = self.conditional_sample(
296
+ condition_data=cond_data,
297
+ condition_mask=cond_mask,
298
+ local_cond=None,
299
+ global_cond=global_cond,
300
+ **self.kwargs)
301
+
302
+ # unnormalize prediction
303
+ assert nsample.shape == (B, self.action_horizon, self.action_dim)
304
+ action_pred = self.normalizer['action'].unnormalize(nsample)
305
+
306
+ result = {
307
+ 'action': action_pred,
308
+ 'action_pred': action_pred
309
+ }
310
+ return result
311
+
312
+ # ========= training ============
313
+ def set_normalizer(self, normalizer: LinearNormalizer):
314
+ self.normalizer.load_state_dict(normalizer.state_dict())
315
+
316
+ def compute_loss(self, batch):
317
+ # normalize input
318
+ assert 'valid_mask' not in batch
319
+ nobs = self.normalizer.normalize(batch['obs'])
320
+ nactions_dataset = self.normalizer['action'].normalize(batch['action'])
321
+
322
+ # ---------------- PA-RL action-optimization target ----------------
323
+ # When the dataloader emits ``action_optimized`` (UmiDataset's
324
+ # cache populated by the workspace's relabel pass), swap the
325
+ # DDPM target from the dataset action to the optimized action
326
+ # on a PER-SAMPLE basis. ``has_action_optimized`` is the gate
327
+ # (1.0 => optimized; 0.0 => dataset fallback), mirroring the
328
+ # ``has_advantage`` convention so partially-cached batches don't
329
+ # silently zero out a fraction of samples.
330
+ #
331
+ # PA-RL is mutually exclusive with AWR within a single batch:
332
+ # we don't want to "Q-optimize the action then re-weight the
333
+ # loss on it by stale dataset advantages". When PA-RL fires,
334
+ # the AWR knobs are forced inactive and the loss reduces to
335
+ # the standard DDPM denoising loss against the optimized chunk
336
+ # plus any optional anchor terms. The PA-RL config sets
337
+ # ``awr_beta=0`` for the same reason; this gate is just defense
338
+ # in depth.
339
+ action_optimized = batch.get('action_optimized', None)
340
+ has_action_optimized = batch.get('has_action_optimized', None)
341
+ parl_active = (
342
+ action_optimized is not None
343
+ and has_action_optimized is not None
344
+ and bool((has_action_optimized > 0.5).any().item())
345
+ )
346
+ if parl_active:
347
+ nactions_opt = self.normalizer['action'].normalize(action_optimized)
348
+ has_opt = has_action_optimized.view(-1, 1, 1).to(
349
+ dtype=torch.bool, device=nactions_opt.device)
350
+ nactions = torch.where(has_opt, nactions_opt, nactions_dataset)
351
+ else:
352
+ nactions = nactions_dataset
353
+
354
+ # AWR per-sample inputs. Pulled here (BEFORE the optional
355
+ # train_diffusion_n_samples expansion below) so we can repeat
356
+ # them in lockstep with the action / obs duplication. Both
357
+ # default to None when the batch doesn't carry advantage info,
358
+ # which is also the path taken when awr_beta <= 0.
359
+ # When PA-RL is active we force these to None so the AWR
360
+ # weighting path is fully off (see comment block above).
361
+ advantage = batch.get('advantage', None) if not parl_active else None
362
+ has_advantage = batch.get('has_advantage', None) if not parl_active else None
363
+ awr_active = (
364
+ self.awr_beta > 0.0
365
+ and advantage is not None
366
+ and has_advantage is not None
367
+ )
368
+
369
+ assert self.obs_as_global_cond
370
+ global_cond = self.obs_encoder(nobs)
371
+
372
+ # train on multiple diffusion samples per obs
373
+ if self.train_diffusion_n_samples != 1:
374
+ # repeat obs features and actions multiple times along the batch dimension
375
+ # each sample will later have a different noise sample, effecty training
376
+ # more diffusion steps per each obs encoder forward pass
377
+ global_cond = torch.repeat_interleave(global_cond,
378
+ repeats=self.train_diffusion_n_samples, dim=0)
379
+ nactions = torch.repeat_interleave(nactions,
380
+ repeats=self.train_diffusion_n_samples, dim=0)
381
+ if awr_active:
382
+ advantage = torch.repeat_interleave(
383
+ advantage, repeats=self.train_diffusion_n_samples, dim=0)
384
+ has_advantage = torch.repeat_interleave(
385
+ has_advantage, repeats=self.train_diffusion_n_samples, dim=0)
386
+
387
+ trajectory = nactions
388
+ # Sample noise that we'll add to the images
389
+ noise = torch.randn(trajectory.shape, device=trajectory.device)
390
+ # input perturbation by adding additonal noise to alleviate exposure bias
391
+ # reference: https://github.com/forever208/DDPM-IP
392
+ noise_new = noise + self.input_pertub * torch.randn(trajectory.shape, device=trajectory.device)
393
+
394
+ # Sample a random timestep for each image
395
+ timesteps = torch.randint(
396
+ 0, self.noise_scheduler.config.num_train_timesteps,
397
+ (nactions.shape[0],), device=trajectory.device
398
+ ).long()
399
+
400
+ # Add noise to the clean images according to the noise magnitude at each timestep
401
+ # (this is the forward diffusion process)
402
+ noisy_trajectory = self.noise_scheduler.add_noise(
403
+ trajectory, noise_new, timesteps)
404
+
405
+ # Predict the noise residual
406
+ pred = self.model(
407
+ noisy_trajectory,
408
+ timesteps,
409
+ local_cond=None,
410
+ global_cond=global_cond
411
+ )
412
+
413
+ pred_type = self.noise_scheduler.config.prediction_type
414
+ if pred_type == 'epsilon':
415
+ target = noise
416
+ elif pred_type == 'sample':
417
+ target = trajectory
418
+ else:
419
+ raise ValueError(f"Unsupported prediction type {pred_type}")
420
+
421
+ loss = F.mse_loss(pred, target, reduction='none')
422
+ loss = loss.type(loss.dtype)
423
+ # Reduce to a single scalar per sample (B,). We keep this
424
+ # un-meaned so AWR can apply its per-sample weight cleanly.
425
+ loss_per_sample = reduce(loss, 'b ... -> b', 'mean')
426
+
427
+ # Compute the anchor loss once (independent of AWR weighting)
428
+ # so we can add it to either the BC-only or the AWR path.
429
+ # Skipped when no frozen reference is attached, i.e. when
430
+ # `awr_anchor_lambda` is 0 or `init_frozen_ref()` was never
431
+ # called -- preserving byte-identical legacy BC behavior.
432
+ loss_anchor = None
433
+ if (
434
+ self.awr_anchor_lambda > 0.0
435
+ and self.frozen_ref_model is not None
436
+ ):
437
+ # The frozen ref is intentionally registered via
438
+ # `object.__setattr__` (NOT as an nn.Module submodule) so
439
+ # it's invisible to DDP / EMA / the optimizer. The
440
+ # downside: `accelerator.prepare(self.model)` / `.to()`
441
+ # don't recurse into it, so on the first forward it may
442
+ # still be on CPU while the rest of the model is on the
443
+ # current process's CUDA device. Move it ONCE here --
444
+ # idempotent on subsequent steps and free when the device
445
+ # already matches.
446
+ ref_device = next(
447
+ self.frozen_ref_model.parameters()).device
448
+ if ref_device != noisy_trajectory.device:
449
+ self.frozen_ref_model.to(noisy_trajectory.device)
450
+ with torch.no_grad():
451
+ pred_ref = self.frozen_ref_model(
452
+ noisy_trajectory, timesteps,
453
+ local_cond=None, global_cond=global_cond,
454
+ )
455
+ loss_anchor = F.mse_loss(pred, pred_ref)
456
+
457
+ if not awr_active:
458
+ # Legacy BC path: byte-identical to the pre-AWR loss
459
+ # (mean over the per-sample MSE scalar is mathematically
460
+ # equivalent to mean over the full (b, h, a) tensor).
461
+ # Still permit the anchor term to fire here so
462
+ # `awr_beta=0, awr_anchor_lambda>0` gives plain BC-with-
463
+ # anchor (useful as a sanity baseline).
464
+ self.last_awr_info = {}
465
+ total = loss_per_sample.mean()
466
+ if loss_anchor is not None:
467
+ total = total + self.awr_anchor_lambda * loss_anchor
468
+ self.last_awr_info = {
469
+ 'awr/loss_bc': float(loss_per_sample.mean()),
470
+ 'awr/loss_anchor': float(loss_anchor),
471
+ }
472
+ # PA-RL diagnostics: track what fraction of the batch was
473
+ # sourced from the optimized cache vs fell back to dataset
474
+ # actions. Mean L2 between optimized and dataset target
475
+ # tells us whether the PA-RL relabel pass actually moved
476
+ # actions (vs collapsed back to the BC chunk).
477
+ if parl_active:
478
+ with torch.no_grad():
479
+ has_opt_b = has_action_optimized.float().view(-1)
480
+ n_total = int(has_opt_b.numel())
481
+ frac_opt = float(has_opt_b.mean().item()) \
482
+ if n_total > 0 else 0.0
483
+ if has_opt_b.bool().any():
484
+ delta = (
485
+ nactions_opt[has_opt_b.bool()]
486
+ - nactions_dataset[has_opt_b.bool()]
487
+ ).flatten(start_dim=1)
488
+ l2 = float(delta.norm(dim=-1).mean().item())
489
+ else:
490
+ l2 = 0.0
491
+ self.last_awr_info.update({
492
+ 'parl/active': 1.0,
493
+ 'parl/frac_optimized': frac_opt,
494
+ 'parl/target_l2_change_norm_mean': l2,
495
+ 'parl/loss_ddpm': float(loss_per_sample.mean().item()),
496
+ 'parl/loss_anchor': (
497
+ float(loss_anchor.item())
498
+ if loss_anchor is not None else 0.0
499
+ ),
500
+ })
501
+ return total
502
+
503
+ # ===== AWR weighting =====
504
+ # advantage / has_advantage are detached float tensors from the
505
+ # data loader, but be paranoid about dtype/device alignment in
506
+ # case a future loader change supplies them in fp16 / cpu.
507
+ adv = advantage.detach().to(
508
+ device=loss_per_sample.device, dtype=loss_per_sample.dtype)
509
+ m = has_advantage.detach().to(
510
+ device=loss_per_sample.device, dtype=loss_per_sample.dtype)
511
+
512
+ # Numerically safe exp: clamp the exponent so a single sample
513
+ # with a very large advantage can't blow up to inf. We clamp
514
+ # in *log space* against awr_w_max so the post-clamp weight is
515
+ # exactly the same as `clamp(exp(...), max=awr_w_max)` but we
516
+ # never materialize the inf intermediate.
517
+ log_w_cap = float(np.log(self.awr_w_max))
518
+ exponent = (adv / self.awr_beta).clamp(max=log_w_cap)
519
+ w_awr = torch.exp(exponent) # (B,), in (0, awr_w_max]
520
+
521
+ # Optional lower clamp on the per-sample weight, applied BEFORE
522
+ # the "no advantage info -> w=1" fallback so low-advantage
523
+ # but on-distribution samples can't be silently zeroed by a
524
+ # very low (or very negative) `adv / beta`. Acts as a soft
525
+ # "always respect at least this much of BC" prior.
526
+ if self.awr_w_min > 0.0:
527
+ w_awr = w_awr.clamp(min=self.awr_w_min)
528
+
529
+ # Replace AWR weight with 1.0 (i.e. plain BC) for samples that
530
+ # were flagged as not having a valid IQL anchor by the dataset.
531
+ # This keeps gradient information flowing through those samples
532
+ # rather than zeroing them out, which would shrink the
533
+ # effective batch size on partially-annotated datasets.
534
+ w = torch.where(m > 0.5, w_awr, torch.ones_like(w_awr))
535
+
536
+ # Optional per-batch normalization. Without it, the absolute
537
+ # magnitude of the loss (and therefore the effective learning
538
+ # rate) drifts with awr_beta and with the empirical advantage
539
+ # distribution of the current batch. We intentionally normalize
540
+ # over the FULL batch (including the BC-fallback samples) so
541
+ # the relative weighting between AWR-eligible and ineligible
542
+ # samples is preserved.
543
+ if self.awr_normalize_weights:
544
+ w = w / (w.mean() + 1e-8)
545
+
546
+ # BC↔AWR linear blend. lambda=1 = pure AWR; lambda=0 collapses
547
+ # to plain BC (useful as a sanity baseline at config flip).
548
+ eff_w = (1.0 - self.awr_lambda) * torch.ones_like(w) \
549
+ + self.awr_lambda * w
550
+
551
+ # The "AWR" term in our 3-term BC-anchored loss.
552
+ loss_awr = (loss_per_sample * eff_w).mean()
553
+
554
+ # Optional UNWEIGHTED BC term ("pull back toward plain
555
+ # imitation of every demo equally"). Uses the SAME per-sample
556
+ # MSE we already computed -- no extra forward pass.
557
+ loss_bc = loss_per_sample.mean()
558
+
559
+ total = loss_awr
560
+ if self.awr_bc_lambda > 0.0:
561
+ total = total + self.awr_bc_lambda * loss_bc
562
+ if loss_anchor is not None:
563
+ total = total + self.awr_anchor_lambda * loss_anchor
564
+
565
+ # Per-batch diagnostics. Stashed on the module so the workspace
566
+ # can pull them into wandb without changing compute_loss's
567
+ # contract (still returns a single scalar).
568
+ with torch.no_grad():
569
+ valid = m > 0.5
570
+ n_valid = valid.sum().clamp_min(1)
571
+ adv_valid_mean = (adv * valid).sum() / n_valid
572
+ adv_valid_max = adv[valid].max() if valid.any() else torch.tensor(
573
+ float('nan'), device=adv.device)
574
+ adv_valid_min = adv[valid].min() if valid.any() else torch.tensor(
575
+ float('nan'), device=adv.device)
576
+ self.last_awr_info = {
577
+ 'awr/active': 1.0,
578
+ 'awr/beta': float(self.awr_beta),
579
+ 'awr/lambda': float(self.awr_lambda),
580
+ 'awr/bc_lambda': float(self.awr_bc_lambda),
581
+ 'awr/anchor_lambda': float(self.awr_anchor_lambda),
582
+ 'awr/w_min_floor': float(self.awr_w_min),
583
+ 'awr/w_mean': float(w.mean()),
584
+ 'awr/w_std': float(w.std()),
585
+ 'awr/w_min': float(w.min()),
586
+ 'awr/w_max_seen': float(w.max()),
587
+ 'awr/eff_w_mean': float(eff_w.mean()),
588
+ 'awr/eff_w_std': float(eff_w.std()),
589
+ 'awr/frac_clipped': float(
590
+ (exponent >= log_w_cap - 1e-6).float().mean()),
591
+ 'awr/frac_floored': (
592
+ float((w_awr <= self.awr_w_min + 1e-6).float().mean())
593
+ if self.awr_w_min > 0.0 else 0.0),
594
+ 'awr/frac_has_adv': float(valid.float().mean()),
595
+ 'awr/adv_valid_mean': float(adv_valid_mean),
596
+ 'awr/adv_valid_min': float(adv_valid_min),
597
+ 'awr/adv_valid_max': float(adv_valid_max),
598
+ 'awr/loss_awr': float(loss_awr),
599
+ 'awr/loss_bc': float(loss_bc),
600
+ 'awr/loss_anchor': (
601
+ float(loss_anchor) if loss_anchor is not None else 0.0),
602
+ }
603
+
604
+ return total
605
+
606
+ def forward(self, batch):
607
+ return self.compute_loss(batch)