solarian-diffusion 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. diffusion_policy/__init__.py +0 -0
  2. diffusion_policy/codecs/__init__.py +0 -0
  3. diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
  4. diffusion_policy/common/__init__.py +0 -0
  5. diffusion_policy/common/checkpoint_util.py +64 -0
  6. diffusion_policy/common/json_logger.py +117 -0
  7. diffusion_policy/common/normalize_util.py +251 -0
  8. diffusion_policy/common/pose_repr_util.py +112 -0
  9. diffusion_policy/common/pytorch_util.py +82 -0
  10. diffusion_policy/common/q_critic_util.py +135 -0
  11. diffusion_policy/common/replay_buffer.py +672 -0
  12. diffusion_policy/common/sampler.py +204 -0
  13. diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
  14. diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
  15. diffusion_policy/config/legacy/task/can_image.yaml +64 -0
  16. diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
  17. diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
  18. diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
  19. diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
  20. diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
  21. diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
  22. diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
  23. diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
  24. diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
  25. diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
  26. diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
  27. diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
  28. diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
  29. diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
  30. diffusion_policy/config/legacy/task/square_image.yaml +64 -0
  31. diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
  32. diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
  33. diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
  34. diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
  35. diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
  36. diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
  37. diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
  38. diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
  39. diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
  40. diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
  41. diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
  42. diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
  43. diffusion_policy/config/legacy/task/umi.yaml +45 -0
  44. diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
  45. diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
  46. diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
  47. diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
  48. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
  49. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
  50. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
  51. diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
  52. diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
  53. diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
  54. diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
  55. diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
  56. diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
  57. diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
  58. diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
  59. diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
  60. diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
  61. diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
  62. diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
  63. diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
  64. diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
  65. diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
  66. diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
  67. diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
  68. diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
  69. diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
  70. diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
  71. diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
  72. diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
  73. diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
  74. diffusion_policy/config/rl_annotate.yaml +265 -0
  75. diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
  76. diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
  77. diffusion_policy/config/rl_annotate_v2.yaml +141 -0
  78. diffusion_policy/config/rl_annotate_v3.yaml +165 -0
  79. diffusion_policy/config/task/lift.yaml +89 -0
  80. diffusion_policy/config/task/square.yaml +92 -0
  81. diffusion_policy/config/task/tool_hang.yaml +93 -0
  82. diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
  83. diffusion_policy/config/task/umi.yaml +105 -0
  84. diffusion_policy/config/task/umi_bimanual.yaml +135 -0
  85. diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
  86. diffusion_policy/config/task/umi_image.yaml +36 -0
  87. diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
  88. diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
  89. diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
  90. diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
  91. diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
  92. diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
  93. diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
  94. diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
  95. diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
  96. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
  97. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
  98. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
  99. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
  100. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
  101. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
  102. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
  103. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
  104. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
  105. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
  106. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
  107. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
  108. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
  109. diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
  110. diffusion_policy/config/train_iql_umi.yaml +186 -0
  111. diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
  112. diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
  113. diffusion_policy/dataset/__init__.py +0 -0
  114. diffusion_policy/dataset/base_dataset.py +76 -0
  115. diffusion_policy/dataset/umi_dataset.py +652 -0
  116. diffusion_policy/env_runner/__init__.py +0 -0
  117. diffusion_policy/env_runner/base_image_runner.py +9 -0
  118. diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
  119. diffusion_policy/model/__init__.py +0 -0
  120. diffusion_policy/model/common/__init__.py +0 -0
  121. diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
  122. diffusion_policy/model/common/lr_scheduler.py +46 -0
  123. diffusion_policy/model/common/module_attr_mixin.py +14 -0
  124. diffusion_policy/model/common/normalizer.py +353 -0
  125. diffusion_policy/model/common/tensor_util.py +960 -0
  126. diffusion_policy/model/diffusion/__init__.py +0 -0
  127. diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
  128. diffusion_policy/model/diffusion/conv1d_components.py +46 -0
  129. diffusion_policy/model/diffusion/ema_model.py +88 -0
  130. diffusion_policy/model/diffusion/mask_generator.py +221 -0
  131. diffusion_policy/model/diffusion/positional_embedding.py +17 -0
  132. diffusion_policy/model/vision/__init__.py +0 -0
  133. diffusion_policy/model/vision/crop_randomizer.py +288 -0
  134. diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
  135. diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
  136. diffusion_policy/policy/__init__.py +0 -0
  137. diffusion_policy/policy/base_image_policy.py +27 -0
  138. diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
  139. diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
  140. diffusion_policy/policy/parl_action_optimizer.py +675 -0
  141. diffusion_policy/workspace/__init__.py +0 -0
  142. diffusion_policy/workspace/base_workspace.py +146 -0
  143. diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
  144. solarian_diffusion/__init__.py +32 -0
  145. solarian_diffusion/config.py +31 -0
  146. solarian_diffusion/policy_server.py +316 -0
  147. solarian_diffusion/rtc.py +272 -0
  148. solarian_diffusion/service.py +16 -0
  149. solarian_diffusion/trainer.py +213 -0
  150. solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
  151. solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
  152. solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
  153. solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
  154. solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
  155. solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
  156. solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
  157. umi/__init__.py +0 -0
  158. umi/common/__init__.py +0 -0
  159. umi/common/pose_util.py +115 -0
@@ -0,0 +1,675 @@
1
+ """PA-RL (Policy-Agnostic RL) action optimization.
2
+
3
+ Implements the action-optimization stage of PA-RL (Sobol Mark et al.,
4
+ 2024, "Policy Agnostic RL", arXiv:2412.06685):
5
+
6
+ 1. Propose K action chunks per state by sampling from the current
7
+ diffusion policy.
8
+ 2. Global optimization: rerank by the IQL critic and keep the top-m.
9
+ 3. Local optimization: T gradient ascent steps on each kept chunk
10
+ w.r.t. min(Q1, Q2), with a per-step revert-on-no-improve guard.
11
+ 4. Pick: argmax-Q or one sample from softmax(Q / tau) over the
12
+ optimized chunks (paper's recommendation: argmax when the
13
+ intra-batch Q-variance is small, softmax otherwise; a single
14
+ ``mode`` knob picks per call).
15
+
16
+ The whole pipeline is a *pure function over a frozen IQL critic*: no
17
+ parameter updates, no global state. That makes it equally usable from
18
+
19
+ * the relabel pass in ``TrainDiffusionUnetImageWorkspace`` (offline
20
+ PA-RL distillation), and
21
+ * future inference-time online PA-RL, should we add it.
22
+
23
+ Performance notes:
24
+ * The visual encoder is the dominant cost. We compute
25
+ ``critic.state_embed(obs)`` ONCE per batch of states and reuse it
26
+ for both global rerank and every step of local gradient ascent
27
+ (mirrors ``policy_server._q_min_from_state_embed``).
28
+ * Chunk sampling expands obs along a fake "K" axis so the diffusion
29
+ policy's U-Net sees a single batched tensor of shape
30
+ ``(B*K, H, A)``. The encoder runs once (with B*K rows of repeated
31
+ obs); the U-Net's per-sample noise gives the K candidates their
32
+ diversity.
33
+ * Local optimization runs on UNNORMALIZED actions (the critic owns
34
+ its own ``LinearNormalizer`` and re-normalizes inside
35
+ ``action_embed``). This keeps the gradient direction interpretable
36
+ in robot-space units, which matters because the Q-net was trained
37
+ on the same unnormalized action representation.
38
+ """
39
+
40
+ from __future__ import annotations
41
+
42
+ import logging
43
+ from dataclasses import dataclass
44
+ from typing import Any, Dict, List, Optional, Tuple
45
+
46
+ import torch
47
+
48
+ logger = logging.getLogger(__name__)
49
+
50
+
51
+ # ---------------------------------------------------------------------------
52
+ # Q-min helper: avoid re-running the IQL obs encoder per candidate.
53
+ # ---------------------------------------------------------------------------
54
+
55
+
56
+ def q_min_from_state_embed(
57
+ critic,
58
+ state_embed: torch.Tensor,
59
+ action: torch.Tensor,
60
+ ) -> torch.Tensor:
61
+ """``min(Q1, Q2)`` given a precomputed ``state_embed`` and an
62
+ UNNORMALIZED action chunk.
63
+
64
+ Identical contract to ``policy_server._q_min_from_state_embed``;
65
+ duplicated here so this module has no dependency on the FastAPI
66
+ server. ``critic.action_embed(...)`` re-normalizes internally.
67
+
68
+ Shapes:
69
+ state_embed: (N, D)
70
+ action: (N, H, A) -- unnormalized
71
+ Returns:
72
+ (N,) tensor.
73
+ """
74
+ a = critic.action_embed(action, target=False)
75
+ sa = torch.cat([state_embed, a], dim=-1)
76
+ q1 = critic.q1_head(sa).squeeze(-1)
77
+ q2 = critic.q2_head(sa).squeeze(-1)
78
+ return torch.minimum(q1, q2)
79
+
80
+
81
+ # ---------------------------------------------------------------------------
82
+ # Stage I.a: proposal — sample K chunks per state from the diffusion policy.
83
+ # ---------------------------------------------------------------------------
84
+
85
+
86
+ @torch.no_grad()
87
+ def propose_actions(
88
+ proposer,
89
+ obs_dict: Dict[str, torch.Tensor],
90
+ K: int,
91
+ *,
92
+ inference_steps: Optional[int] = None,
93
+ global_cond_override: Optional[torch.Tensor] = None,
94
+ ) -> torch.Tensor:
95
+ """Sample K UNNORMALIZED action chunks per state.
96
+
97
+ Implements Equation 4.1's "candidate set"
98
+ :math:`\\mathcal{A}_{\\pi_\\varphi, K}(s)` for diffusion policies.
99
+
100
+ Performance note: the visual encoder (typically a frozen ViT-L)
101
+ is the dominant cost of this call. Naively expanding the obs to
102
+ ``(B*K, ...)`` before the encoder runs the (heavy, O(image)) ViT
103
+ K times on each unique state. We instead:
104
+
105
+ 1. encode the B UNIQUE obs once -> ``global_cond_B: (B, D)``
106
+ 2. repeat-interleave the encoded vector to ``(B*K, D)``
107
+ 3. run the (lightweight, O(traj)) U-Net at batch B*K with K
108
+ independent noise draws
109
+
110
+ This is mathematically identical to the naive path because the
111
+ encoder is row-independent, but cuts encoder cost K-fold (32x
112
+ at the paper's default K=32). The U-Net cost is unchanged
113
+ because each candidate still gets its own noise sample at every
114
+ DDIM step.
115
+
116
+ Args:
117
+ proposer: a ``DiffusionUnetTimmPolicy``-shaped object exposing
118
+ ``normalizer``, ``obs_encoder``, ``action_horizon``,
119
+ ``action_dim``, ``device``, ``dtype``,
120
+ ``num_inference_steps``, and ``conditional_sample`` /
121
+ ``noise_scheduler`` / ``model``. The on-policy snapshot
122
+ (e.g. EMA model) the relabel pass uses.
123
+ obs_dict: ``(B, ...)`` per key, raw (unnormalized) obs.
124
+ K: number of chunks per state.
125
+ inference_steps: temporary override of
126
+ ``proposer.num_inference_steps`` for the duration of this
127
+ call. Useful to run cheaper sampling during relabel
128
+ (e.g. 5 DDIM steps) than the policy uses at deployment
129
+ (e.g. 16). Mutating the attribute is safe because we
130
+ ALWAYS restore the original in a try/finally and we hold
131
+ ``torch.no_grad`` so no autograd state is tied to the
132
+ attribute. Set to ``None`` to use ``proposer.num_inference_steps``.
133
+ global_cond_override: optional precomputed ``(B, D_cond)``
134
+ tensor produced by the SAME ``proposer.obs_encoder`` for
135
+ the SAME ``obs_dict``. When provided, we skip both the
136
+ normalize and obs_encoder forwards entirely. Intended for
137
+ workflows that pre-compute and cache the encoder output
138
+ once for all dataset frames (the proposer's encoder is
139
+ frozen when ``training.freeze_encoder=True``, so its
140
+ output is invariant across relabel cycles). Must be on
141
+ the same device as the U-Net; dtype is preserved.
142
+
143
+ Returns:
144
+ ``(B, K, H, A)`` tensor on ``proposer.device``, UNNORMALIZED
145
+ (matches the contract of ``policy.predict_action``).
146
+ """
147
+ if K <= 0:
148
+ raise ValueError(f"K must be >= 1, got {K}")
149
+
150
+ B = next(iter(obs_dict.values())).shape[0]
151
+
152
+ if global_cond_override is not None:
153
+ # Cache hit: skip normalize + obs_encoder. Validate shape.
154
+ if global_cond_override.shape[0] != B:
155
+ raise ValueError(
156
+ f"global_cond_override has B="
157
+ f"{global_cond_override.shape[0]} but obs_dict has B={B}"
158
+ )
159
+ global_cond_B = global_cond_override
160
+ else:
161
+ nobs = proposer.normalizer.normalize(obs_dict)
162
+ # Encode once on the B unique obs (NOT B*K) — the K candidates
163
+ # per state share identical visual conditioning; only the U-Net
164
+ # noise differs across replicas. Saves K-fold compute on the
165
+ # visual encoder, which is the dominant relabel cost.
166
+ global_cond_B = proposer.obs_encoder(nobs) # (B, D_cond)
167
+ if K > 1:
168
+ global_cond = global_cond_B.repeat_interleave(K, dim=0)
169
+ else:
170
+ global_cond = global_cond_B
171
+
172
+ saved_steps = proposer.num_inference_steps
173
+ if inference_steps is not None:
174
+ proposer.num_inference_steps = int(inference_steps)
175
+ try:
176
+ cond_data = torch.zeros(
177
+ size=(B * K, proposer.action_horizon, proposer.action_dim),
178
+ device=proposer.device, dtype=proposer.dtype,
179
+ )
180
+ cond_mask = torch.zeros_like(cond_data, dtype=torch.bool)
181
+ nsample = proposer.conditional_sample(
182
+ condition_data=cond_data,
183
+ condition_mask=cond_mask,
184
+ local_cond=None,
185
+ global_cond=global_cond,
186
+ )
187
+ finally:
188
+ proposer.num_inference_steps = saved_steps
189
+
190
+ actions = proposer.normalizer["action"].unnormalize(nsample)
191
+ # (B*K, H, A) -> (B, K, H, A) without copying when K=1.
192
+ actions = actions.view(B, K, proposer.action_horizon, proposer.action_dim)
193
+ return actions
194
+
195
+
196
+ # ---------------------------------------------------------------------------
197
+ # Stage I.b: global optimization — rerank K candidates, keep top-m.
198
+ # ---------------------------------------------------------------------------
199
+
200
+
201
+ @torch.no_grad()
202
+ def global_optimize(
203
+ critic,
204
+ state_embed_K: torch.Tensor,
205
+ candidates: torch.Tensor,
206
+ m: int,
207
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
208
+ """Equation 4.1: rerank K candidates per state and keep top-m.
209
+
210
+ Args:
211
+ critic: IQL critic (provides ``action_embed``, ``q1_head``,
212
+ ``q2_head``).
213
+ state_embed_K: (B*K, D) state embedding, one row per
214
+ candidate. Caller must already have repeated the per-state
215
+ embedding K times (so the critic doesn't re-encode obs per
216
+ candidate).
217
+ candidates: (B, K, H, A) UNNORMALIZED chunks.
218
+ m: number of top candidates to retain. ``m <= K`` enforced.
219
+
220
+ Returns:
221
+ ``(top_actions, top_q)`` where
222
+ top_actions: (B, m, H, A)
223
+ top_q: (B, m) -- min(Q1, Q2) of each kept chunk.
224
+ """
225
+ B, K, H, A = candidates.shape
226
+ if m <= 0:
227
+ raise ValueError(f"m must be >= 1, got {m}")
228
+ if m > K:
229
+ raise ValueError(f"m ({m}) > K ({K})")
230
+
231
+ flat = candidates.reshape(B * K, H, A)
232
+ q_min = q_min_from_state_embed(critic, state_embed_K, flat)
233
+ q_min = q_min.view(B, K)
234
+
235
+ # ``torch.topk`` returns the m highest values and their indices
236
+ # along the K dim. The sort is stable enough for our purposes
237
+ # (ties are vanishingly unlikely on float32 Q).
238
+ top_q, top_idx = q_min.topk(k=m, dim=1, largest=True, sorted=True)
239
+ # Gather the corresponding chunks. expand+gather avoids the
240
+ # advanced-indexing copy.
241
+ idx_exp = top_idx.view(B, m, 1, 1).expand(B, m, H, A)
242
+ top_actions = candidates.gather(dim=1, index=idx_exp).contiguous()
243
+ return top_actions, top_q
244
+
245
+
246
+ # ---------------------------------------------------------------------------
247
+ # Stage I.c: local optimization — T grad ascent steps with revert-on-no-improve.
248
+ # ---------------------------------------------------------------------------
249
+
250
+
251
+ def local_optimize(
252
+ critic,
253
+ state_embed: torch.Tensor,
254
+ actions: torch.Tensor,
255
+ *,
256
+ T: int,
257
+ alpha: float,
258
+ ) -> Tuple[torch.Tensor, torch.Tensor, Dict[str, float]]:
259
+ """Algorithm 1: T steps of gradient ascent on Q w.r.t. action.
260
+
261
+ For each (state, candidate) pair we do up to T steps of
262
+ ``a_{j+1} = a_j + alpha * grad_a Q(s, a_j)``. After each step we
263
+ re-evaluate Q; if it did NOT improve, we REVERT to the previous
264
+ action and stop early for that candidate (matching the
265
+ "revert-on-no-improve" rule in Algorithm 1 lines 6-9 -- "we only
266
+ execute a local update if it increases the Q-value", Section 4.3).
267
+
268
+ Crucially: we do NOT re-run the obs encoder during this loop. The
269
+ state embedding is computed once by the caller and passed in,
270
+ while ``critic.action_embed`` (a tiny MLP over the flattened
271
+ chunk) runs each step.
272
+
273
+ Args:
274
+ critic: frozen IQL critic.
275
+ state_embed: (N, D) -- one state-embed per candidate.
276
+ Caller is responsible for replicating per-state embeds to
277
+ match the candidate count.
278
+ actions: (N, H, A) UNNORMALIZED candidates. Returned tensor
279
+ has the same shape; gradient flow does NOT escape this
280
+ function.
281
+ T: max number of gradient steps per candidate.
282
+ alpha: gradient step size in unnormalized action units. The
283
+ paper uses 3e-4 for state-based tasks; image-based PA-RL
284
+ uses the same default.
285
+
286
+ Returns:
287
+ ``(actions_opt, q_after, info)`` where
288
+ actions_opt: (N, H, A)
289
+ q_after: (N,) -- final ``min(Q1, Q2)``.
290
+ info: aggregate diagnostics (q_before mean, q_after mean,
291
+ q_gain mean, frac_local_reverted, action_l2_change).
292
+ """
293
+ if actions.dim() != 3:
294
+ raise ValueError(
295
+ f"local_optimize expects (N, H, A) actions; got {actions.shape}")
296
+ N = actions.shape[0]
297
+ info: Dict[str, float] = {
298
+ "n_candidates": int(N),
299
+ "T": int(T),
300
+ "alpha": float(alpha),
301
+ }
302
+
303
+ if N == 0 or T <= 0 or alpha <= 0.0:
304
+ with torch.no_grad():
305
+ q_now = q_min_from_state_embed(critic, state_embed, actions)
306
+ info.update({
307
+ "q_before_mean": float(q_now.mean().item()),
308
+ "q_after_mean": float(q_now.mean().item()),
309
+ "q_gain_mean": 0.0,
310
+ "frac_local_reverted": 0.0,
311
+ "action_l2_change_mean": 0.0,
312
+ })
313
+ return actions.detach(), q_now.detach(), info
314
+
315
+ # Gradient ascent. We maintain a "current best" copy and only
316
+ # replace it on a Q increase; this is the exact rule in Algorithm 1
317
+ # which the paper treats as the safe knob ("not guaranteed to
318
+ # improve... we only execute a local update if it increases the
319
+ # Q-value", Sec 4.3).
320
+ actions_curr = actions.detach().clone()
321
+ with torch.no_grad():
322
+ q_before = q_min_from_state_embed(critic, state_embed, actions_curr)
323
+ q_curr = q_before.clone()
324
+
325
+ # Track which candidates are still "live". Once a candidate
326
+ # reverts, we freeze it for the rest of the loop (Algorithm 1's
327
+ # explicit `Break`).
328
+ alive = torch.ones(N, dtype=torch.bool, device=actions.device)
329
+ n_steps_used = torch.zeros(N, dtype=torch.long, device=actions.device)
330
+
331
+ for step_idx in range(int(T)):
332
+ if not alive.any():
333
+ break
334
+ # Build a candidate tensor with grads only on the live rows;
335
+ # autograd is happy if the dead rows simply don't contribute
336
+ # to the loss we sum over.
337
+ with torch.enable_grad():
338
+ a_var = actions_curr.detach().clone().requires_grad_(True)
339
+ q_now = q_min_from_state_embed(critic, state_embed, a_var)
340
+ # Sum only the live rows so we don't compute dead-row grads.
341
+ (q_now * alive.float()).sum().backward()
342
+ grad = a_var.grad
343
+ # Per-candidate proposed update. Dead rows get zeroed in-place
344
+ # so they don't contaminate the next iteration.
345
+ proposed = actions_curr + float(alpha) * grad.detach()
346
+ # Re-eval Q at the proposed actions. ``no_grad`` is fine; we
347
+ # just need the scalar comparison.
348
+ with torch.no_grad():
349
+ q_proposed = q_min_from_state_embed(critic, state_embed, proposed)
350
+ improved = (q_proposed > q_curr) & alive
351
+ # Where improved: accept; else: revert (keep actions_curr,
352
+ # q_curr) and mark dead for subsequent steps.
353
+ if improved.any():
354
+ improved_exp = improved.view(N, 1, 1)
355
+ actions_curr = torch.where(
356
+ improved_exp, proposed, actions_curr,
357
+ )
358
+ q_curr = torch.where(improved, q_proposed, q_curr)
359
+ n_steps_used = torch.where(
360
+ improved, n_steps_used + 1, n_steps_used,
361
+ )
362
+ # Mark non-improved (and previously-alive) rows dead.
363
+ alive = alive & improved
364
+
365
+ # Final stats.
366
+ with torch.no_grad():
367
+ delta = (actions_curr - actions).flatten(start_dim=1)
368
+ l2 = delta.norm(dim=-1)
369
+ # A candidate is "reverted" if it didn't improve at step 0,
370
+ # i.e. it took zero accepted steps and Q is unchanged.
371
+ reverted = (n_steps_used == 0)
372
+ info.update({
373
+ "q_before_mean": float(q_before.mean().item()),
374
+ "q_after_mean": float(q_curr.mean().item()),
375
+ "q_gain_mean": float((q_curr - q_before).mean().item()),
376
+ "frac_local_reverted": float(reverted.float().mean().item()),
377
+ "action_l2_change_mean": float(l2.mean().item()),
378
+ "n_steps_mean": float(n_steps_used.float().mean().item()),
379
+ "n_steps_max": int(n_steps_used.max().item()),
380
+ })
381
+ return actions_curr.detach(), q_curr.detach(), info
382
+
383
+
384
+ # ---------------------------------------------------------------------------
385
+ # Stage II: pick — argmax or softmax(Q/tau) over the optimized set.
386
+ # ---------------------------------------------------------------------------
387
+
388
+
389
+ @dataclass
390
+ class PickResult:
391
+ """Per-state pick output from ``pick_optimized``."""
392
+ action: torch.Tensor # (B, H, A) -- the chosen (and possibly
393
+ # dataset) action for each state.
394
+ chosen_q: torch.Tensor # (B,) -- Q value at the chosen action.
395
+ chose_dataset: torch.Tensor # (B,) bool -- True iff the dataset
396
+ # action was selected for this state.
397
+ info: Dict[str, Any]
398
+
399
+
400
+ def pick_optimized(
401
+ critic,
402
+ state_embed: torch.Tensor,
403
+ optimized_actions: torch.Tensor,
404
+ optimized_q: torch.Tensor,
405
+ *,
406
+ mode: str = "argmax",
407
+ softmax_temperature: float = 1.0,
408
+ dataset_action: Optional[torch.Tensor] = None,
409
+ generator: Optional[torch.Generator] = None,
410
+ ) -> PickResult:
411
+ """Equation 4.3: select ONE action per state from the optimized set.
412
+
413
+ Args:
414
+ critic: frozen IQL critic. Used only when ``dataset_action`` is
415
+ provided, to compute its Q value and merge it into the
416
+ candidate set.
417
+ state_embed: (B, D) one row per state. NOT replicated.
418
+ optimized_actions: (B, m, H, A) the post-local-opt actions.
419
+ optimized_q: (B, m) their min(Q1, Q2) values.
420
+ mode: ``"argmax"`` returns ``argmax_a Q(s, a)``;
421
+ ``"softmax"`` samples a single action per state from
422
+ ``Categorical(logits=Q / tau)``.
423
+ softmax_temperature: tau in the softmax mode. Smaller -> closer
424
+ to argmax; larger -> closer to uniform.
425
+ dataset_action: optional (B, H, A) dataset action chunk to
426
+ include in the candidate set ("we also find that
427
+ additionally including the action a appearing at a given
428
+ state in the dataset into action optimization can sometimes
429
+ be helpful." -- Sec 4.3, Implementation details). If
430
+ provided we evaluate its Q against ``state_embed`` and
431
+ extend the candidate set to (B, m+1, ...).
432
+ generator: optional torch.Generator for reproducible softmax
433
+ sampling.
434
+
435
+ Returns:
436
+ ``PickResult``: ``action`` (B, H, A), ``chosen_q`` (B,),
437
+ ``chose_dataset`` (B,) bool, ``info`` dict.
438
+ """
439
+ B, m, H, A = optimized_actions.shape
440
+ if state_embed.shape[0] != B:
441
+ raise ValueError(
442
+ f"state_embed batch ({state_embed.shape[0]}) != "
443
+ f"optimized_actions batch ({B})"
444
+ )
445
+
446
+ # Optionally extend the candidate set with the dataset action.
447
+ if dataset_action is not None:
448
+ if dataset_action.shape != (B, H, A):
449
+ raise ValueError(
450
+ f"dataset_action shape {tuple(dataset_action.shape)} "
451
+ f"!= expected ({B}, {H}, {A})"
452
+ )
453
+ with torch.no_grad():
454
+ q_data = q_min_from_state_embed(
455
+ critic, state_embed, dataset_action,
456
+ ) # (B,)
457
+ all_actions = torch.cat(
458
+ [optimized_actions, dataset_action.unsqueeze(1)], dim=1,
459
+ ) # (B, m+1, H, A)
460
+ all_q = torch.cat([optimized_q, q_data.unsqueeze(1)], dim=1)
461
+ dataset_idx = m # index of the appended dataset action
462
+ else:
463
+ all_actions = optimized_actions
464
+ all_q = optimized_q
465
+ dataset_idx = None
466
+
467
+ M = all_actions.shape[1]
468
+
469
+ if mode == "argmax":
470
+ chosen = all_q.argmax(dim=1) # (B,)
471
+ elif mode == "softmax":
472
+ if softmax_temperature <= 0:
473
+ raise ValueError(
474
+ f"softmax_temperature must be > 0, got {softmax_temperature}"
475
+ )
476
+ logits = all_q / float(softmax_temperature)
477
+ probs = torch.softmax(logits, dim=1)
478
+ # ``torch.multinomial`` doesn't accept a generator on every
479
+ # backend; fall back to a manual gumbel-max if generator is
480
+ # provided to keep determinism.
481
+ if generator is None:
482
+ chosen = torch.multinomial(probs, num_samples=1).squeeze(1)
483
+ else:
484
+ gumbel = -torch.log(
485
+ -torch.log(
486
+ torch.rand(
487
+ probs.shape, device=probs.device, generator=generator,
488
+ ).clamp_min(1e-12)
489
+ ).clamp_min(1e-12)
490
+ )
491
+ chosen = (logits + gumbel).argmax(dim=1)
492
+ else:
493
+ raise ValueError(
494
+ f"unknown pick mode {mode!r}; expected 'argmax' or 'softmax'"
495
+ )
496
+
497
+ idx_exp = chosen.view(B, 1, 1, 1).expand(B, 1, H, A)
498
+ action = all_actions.gather(dim=1, index=idx_exp).squeeze(1)
499
+ chosen_q = all_q.gather(dim=1, index=chosen.view(B, 1)).squeeze(1)
500
+
501
+ if dataset_idx is None:
502
+ chose_dataset = torch.zeros(B, dtype=torch.bool, device=action.device)
503
+ else:
504
+ chose_dataset = chosen == dataset_idx
505
+
506
+ info: Dict[str, Any] = {
507
+ "mode": mode,
508
+ "softmax_temperature": float(softmax_temperature),
509
+ "n_candidates": int(M),
510
+ "include_dataset_action": dataset_action is not None,
511
+ "frac_dataset_action_chosen": float(chose_dataset.float().mean().item()),
512
+ "q_chosen_mean": float(chosen_q.mean().item()),
513
+ "q_chosen_std": float(chosen_q.std().item()) if B > 1 else 0.0,
514
+ "q_candidates_std_mean": float(all_q.std(dim=1).mean().item())
515
+ if M > 1 else 0.0,
516
+ }
517
+ return PickResult(
518
+ action=action,
519
+ chosen_q=chosen_q,
520
+ chose_dataset=chose_dataset,
521
+ info=info,
522
+ )
523
+
524
+
525
+ # ---------------------------------------------------------------------------
526
+ # End-to-end convenience: propose + global + local + pick.
527
+ # ---------------------------------------------------------------------------
528
+
529
+
530
+ def optimize_actions(
531
+ *,
532
+ proposer,
533
+ critic,
534
+ obs_dict: Dict[str, torch.Tensor],
535
+ K: int,
536
+ m: int,
537
+ T: int,
538
+ alpha: float,
539
+ pick_mode: str = "argmax",
540
+ softmax_temperature: float = 1.0,
541
+ dataset_action: Optional[torch.Tensor] = None,
542
+ proposer_inference_steps: Optional[int] = None,
543
+ generator: Optional[torch.Generator] = None,
544
+ state_embed_override: Optional[torch.Tensor] = None,
545
+ global_cond_override: Optional[torch.Tensor] = None,
546
+ ) -> Tuple[torch.Tensor, Dict[str, Any]]:
547
+ """Run the full PA-RL action optimization pipeline.
548
+
549
+ Sample K -> rerank top-m -> local Q-grad ascent -> pick (argmax /
550
+ softmax / dataset-augmented). The frozen ``critic`` and the
551
+ ``proposer`` (typically a snapshot of the diffusion policy, e.g. the
552
+ EMA model) are used in eval-only mode; this function never touches
553
+ their parameters.
554
+
555
+ Args:
556
+ ... (see other args)
557
+ state_embed_override: optional precomputed ``(B, D)`` state
558
+ embedding produced by the SAME ``critic`` for the SAME
559
+ ``obs_dict``. When provided, we skip
560
+ ``critic.state_embed(obs_dict)`` entirely; intended for
561
+ workflows that pre-compute and cache state embeddings
562
+ once for all dataset frames (the IQL critic's encoder is
563
+ frozen, so its output is invariant across relabel
564
+ cycles). Must be on the same device as the candidate
565
+ actions; dtype is preserved (typically fp16/bf16 if the
566
+ cache was built under autocast).
567
+
568
+ Returns:
569
+ ``(action_opt, info)`` where
570
+ action_opt: (B, H, A) UNNORMALIZED chosen action chunk.
571
+ info: dict of stage-wise diagnostics (each prefixed by
572
+ ``parl/``) suitable for direct logging to wandb.
573
+ """
574
+ if K < 1:
575
+ raise ValueError(f"K must be >= 1, got {K}")
576
+ if m < 1:
577
+ raise ValueError(f"m must be >= 1, got {m}")
578
+ if m > K:
579
+ raise ValueError(f"m ({m}) > K ({K})")
580
+
581
+ B = next(iter(obs_dict.values())).shape[0]
582
+
583
+ # Stage I.a: propose K candidates.
584
+ with torch.no_grad():
585
+ candidates = propose_actions(
586
+ proposer, obs_dict, K,
587
+ inference_steps=proposer_inference_steps,
588
+ global_cond_override=global_cond_override,
589
+ ) # (B, K, H, A)
590
+
591
+ # Compute state_embed ONCE per state (and reuse it across rerank,
592
+ # local-opt, pick). The critic owns its own normalizer + obs
593
+ # encoder; we just feed it the unnormalized obs dict.
594
+ if state_embed_override is not None:
595
+ # Trust the cache: same critic + same obs -> same embedding.
596
+ # We still validate the leading batch dim to fail loudly on
597
+ # off-by-one indexing bugs in the cache lookup site.
598
+ if state_embed_override.shape[0] != B:
599
+ raise ValueError(
600
+ f"state_embed_override has B={state_embed_override.shape[0]} "
601
+ f"but obs_dict has B={B}"
602
+ )
603
+ state_embed = state_embed_override
604
+ used_cache = True
605
+ else:
606
+ with torch.no_grad():
607
+ state_embed = critic.state_embed(obs_dict, target=False) # (B, D)
608
+ used_cache = False
609
+
610
+ # Stage I.b: global optimization. We need state_embed REPEATED to
611
+ # (B*K, D) so the critic scores each candidate independently.
612
+ state_embed_K = state_embed.repeat_interleave(K, dim=0) # (B*K, D)
613
+ top_actions, top_q = global_optimize(
614
+ critic, state_embed_K, candidates, m,
615
+ ) # (B, m, H, A), (B, m)
616
+
617
+ # Stage I.c: local optimization. Same trick: replicate state_embed
618
+ # to (B*m, D).
619
+ state_embed_m = state_embed.repeat_interleave(m, dim=0) # (B*m, D)
620
+ H, A = top_actions.shape[2], top_actions.shape[3]
621
+ flat_actions = top_actions.reshape(B * m, H, A)
622
+ flat_actions_opt, flat_q_after, local_info = local_optimize(
623
+ critic, state_embed_m, flat_actions, T=T, alpha=float(alpha),
624
+ )
625
+ actions_opt = flat_actions_opt.view(B, m, H, A)
626
+ q_after = flat_q_after.view(B, m)
627
+
628
+ # Stage II: pick.
629
+ pick = pick_optimized(
630
+ critic, state_embed, actions_opt, q_after,
631
+ mode=pick_mode,
632
+ softmax_temperature=softmax_temperature,
633
+ dataset_action=dataset_action,
634
+ generator=generator,
635
+ )
636
+
637
+ # Aggregate diagnostics under a parl/ prefix for wandb.
638
+ info: Dict[str, Any] = {
639
+ "parl/K": int(K),
640
+ "parl/m": int(m),
641
+ "parl/T": int(T),
642
+ "parl/alpha": float(alpha),
643
+ "parl/pick_mode": pick_mode,
644
+ "parl/q_global_top1_mean": float(top_q[:, 0].mean().item()),
645
+ "parl/q_global_topm_mean": float(top_q.mean().item()),
646
+ "parl/q_global_spread_mean": float(
647
+ (top_q.max(dim=1).values - top_q.min(dim=1).values).mean().item()
648
+ ) if m > 1 else 0.0,
649
+ "parl/q_before_local_mean": local_info["q_before_mean"],
650
+ "parl/q_after_local_mean": local_info["q_after_mean"],
651
+ "parl/q_local_gain_mean": local_info["q_gain_mean"],
652
+ "parl/frac_local_reverted": local_info["frac_local_reverted"],
653
+ "parl/action_l2_change_mean": local_info["action_l2_change_mean"],
654
+ "parl/n_local_steps_mean": local_info.get("n_steps_mean", 0.0),
655
+ "parl/q_chosen_mean": pick.info["q_chosen_mean"],
656
+ "parl/q_chosen_std": pick.info["q_chosen_std"],
657
+ "parl/q_candidates_std_mean": pick.info["q_candidates_std_mean"],
658
+ "parl/frac_dataset_action_chosen": pick.info["frac_dataset_action_chosen"],
659
+ "parl/include_dataset_action": pick.info["include_dataset_action"],
660
+ "parl/B": int(B),
661
+ "parl/state_embed_cache_hit": float(used_cache),
662
+ "parl/global_cond_cache_hit": float(global_cond_override is not None),
663
+ }
664
+ return pick.action, info
665
+
666
+
667
+ __all__ = [
668
+ "q_min_from_state_embed",
669
+ "propose_actions",
670
+ "global_optimize",
671
+ "local_optimize",
672
+ "pick_optimized",
673
+ "PickResult",
674
+ "optimize_actions",
675
+ ]
File without changes