solarian-diffusion 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffusion_policy/__init__.py +0 -0
- diffusion_policy/codecs/__init__.py +0 -0
- diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
- diffusion_policy/common/__init__.py +0 -0
- diffusion_policy/common/checkpoint_util.py +64 -0
- diffusion_policy/common/json_logger.py +117 -0
- diffusion_policy/common/normalize_util.py +251 -0
- diffusion_policy/common/pose_repr_util.py +112 -0
- diffusion_policy/common/pytorch_util.py +82 -0
- diffusion_policy/common/q_critic_util.py +135 -0
- diffusion_policy/common/replay_buffer.py +672 -0
- diffusion_policy/common/sampler.py +204 -0
- diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
- diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
- diffusion_policy/config/legacy/task/can_image.yaml +64 -0
- diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
- diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
- diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
- diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
- diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
- diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
- diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
- diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
- diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
- diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
- diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
- diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
- diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
- diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
- diffusion_policy/config/legacy/task/square_image.yaml +64 -0
- diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
- diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
- diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
- diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
- diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
- diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
- diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
- diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
- diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
- diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
- diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
- diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
- diffusion_policy/config/legacy/task/umi.yaml +45 -0
- diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
- diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
- diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
- diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
- diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
- diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
- diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
- diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
- diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
- diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
- diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
- diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
- diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
- diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
- diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
- diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
- diffusion_policy/config/rl_annotate.yaml +265 -0
- diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
- diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
- diffusion_policy/config/rl_annotate_v2.yaml +141 -0
- diffusion_policy/config/rl_annotate_v3.yaml +165 -0
- diffusion_policy/config/task/lift.yaml +89 -0
- diffusion_policy/config/task/square.yaml +92 -0
- diffusion_policy/config/task/tool_hang.yaml +93 -0
- diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
- diffusion_policy/config/task/umi.yaml +105 -0
- diffusion_policy/config/task/umi_bimanual.yaml +135 -0
- diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
- diffusion_policy/config/task/umi_image.yaml +36 -0
- diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
- diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
- diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
- diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
- diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
- diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
- diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
- diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
- diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
- diffusion_policy/config/train_iql_umi.yaml +186 -0
- diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
- diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
- diffusion_policy/dataset/__init__.py +0 -0
- diffusion_policy/dataset/base_dataset.py +76 -0
- diffusion_policy/dataset/umi_dataset.py +652 -0
- diffusion_policy/env_runner/__init__.py +0 -0
- diffusion_policy/env_runner/base_image_runner.py +9 -0
- diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
- diffusion_policy/model/__init__.py +0 -0
- diffusion_policy/model/common/__init__.py +0 -0
- diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
- diffusion_policy/model/common/lr_scheduler.py +46 -0
- diffusion_policy/model/common/module_attr_mixin.py +14 -0
- diffusion_policy/model/common/normalizer.py +353 -0
- diffusion_policy/model/common/tensor_util.py +960 -0
- diffusion_policy/model/diffusion/__init__.py +0 -0
- diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
- diffusion_policy/model/diffusion/conv1d_components.py +46 -0
- diffusion_policy/model/diffusion/ema_model.py +88 -0
- diffusion_policy/model/diffusion/mask_generator.py +221 -0
- diffusion_policy/model/diffusion/positional_embedding.py +17 -0
- diffusion_policy/model/vision/__init__.py +0 -0
- diffusion_policy/model/vision/crop_randomizer.py +288 -0
- diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
- diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
- diffusion_policy/policy/__init__.py +0 -0
- diffusion_policy/policy/base_image_policy.py +27 -0
- diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
- diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
- diffusion_policy/policy/parl_action_optimizer.py +675 -0
- diffusion_policy/workspace/__init__.py +0 -0
- diffusion_policy/workspace/base_workspace.py +146 -0
- diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
- solarian_diffusion/__init__.py +32 -0
- solarian_diffusion/config.py +31 -0
- solarian_diffusion/policy_server.py +316 -0
- solarian_diffusion/rtc.py +272 -0
- solarian_diffusion/service.py +16 -0
- solarian_diffusion/trainer.py +213 -0
- solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
- solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
- solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
- solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
- solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
- solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
- solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
- umi/__init__.py +0 -0
- umi/common/__init__.py +0 -0
- umi/common/pose_util.py +115 -0
|
@@ -0,0 +1,675 @@
|
|
|
1
|
+
"""PA-RL (Policy-Agnostic RL) action optimization.
|
|
2
|
+
|
|
3
|
+
Implements the action-optimization stage of PA-RL (Sobol Mark et al.,
|
|
4
|
+
2024, "Policy Agnostic RL", arXiv:2412.06685):
|
|
5
|
+
|
|
6
|
+
1. Propose K action chunks per state by sampling from the current
|
|
7
|
+
diffusion policy.
|
|
8
|
+
2. Global optimization: rerank by the IQL critic and keep the top-m.
|
|
9
|
+
3. Local optimization: T gradient ascent steps on each kept chunk
|
|
10
|
+
w.r.t. min(Q1, Q2), with a per-step revert-on-no-improve guard.
|
|
11
|
+
4. Pick: argmax-Q or one sample from softmax(Q / tau) over the
|
|
12
|
+
optimized chunks (paper's recommendation: argmax when the
|
|
13
|
+
intra-batch Q-variance is small, softmax otherwise; a single
|
|
14
|
+
``mode`` knob picks per call).
|
|
15
|
+
|
|
16
|
+
The whole pipeline is a *pure function over a frozen IQL critic*: no
|
|
17
|
+
parameter updates, no global state. That makes it equally usable from
|
|
18
|
+
|
|
19
|
+
* the relabel pass in ``TrainDiffusionUnetImageWorkspace`` (offline
|
|
20
|
+
PA-RL distillation), and
|
|
21
|
+
* future inference-time online PA-RL, should we add it.
|
|
22
|
+
|
|
23
|
+
Performance notes:
|
|
24
|
+
* The visual encoder is the dominant cost. We compute
|
|
25
|
+
``critic.state_embed(obs)`` ONCE per batch of states and reuse it
|
|
26
|
+
for both global rerank and every step of local gradient ascent
|
|
27
|
+
(mirrors ``policy_server._q_min_from_state_embed``).
|
|
28
|
+
* Chunk sampling expands obs along a fake "K" axis so the diffusion
|
|
29
|
+
policy's U-Net sees a single batched tensor of shape
|
|
30
|
+
``(B*K, H, A)``. The encoder runs once (with B*K rows of repeated
|
|
31
|
+
obs); the U-Net's per-sample noise gives the K candidates their
|
|
32
|
+
diversity.
|
|
33
|
+
* Local optimization runs on UNNORMALIZED actions (the critic owns
|
|
34
|
+
its own ``LinearNormalizer`` and re-normalizes inside
|
|
35
|
+
``action_embed``). This keeps the gradient direction interpretable
|
|
36
|
+
in robot-space units, which matters because the Q-net was trained
|
|
37
|
+
on the same unnormalized action representation.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
import logging
|
|
43
|
+
from dataclasses import dataclass
|
|
44
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
45
|
+
|
|
46
|
+
import torch
|
|
47
|
+
|
|
48
|
+
logger = logging.getLogger(__name__)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# ---------------------------------------------------------------------------
|
|
52
|
+
# Q-min helper: avoid re-running the IQL obs encoder per candidate.
|
|
53
|
+
# ---------------------------------------------------------------------------
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def q_min_from_state_embed(
|
|
57
|
+
critic,
|
|
58
|
+
state_embed: torch.Tensor,
|
|
59
|
+
action: torch.Tensor,
|
|
60
|
+
) -> torch.Tensor:
|
|
61
|
+
"""``min(Q1, Q2)`` given a precomputed ``state_embed`` and an
|
|
62
|
+
UNNORMALIZED action chunk.
|
|
63
|
+
|
|
64
|
+
Identical contract to ``policy_server._q_min_from_state_embed``;
|
|
65
|
+
duplicated here so this module has no dependency on the FastAPI
|
|
66
|
+
server. ``critic.action_embed(...)`` re-normalizes internally.
|
|
67
|
+
|
|
68
|
+
Shapes:
|
|
69
|
+
state_embed: (N, D)
|
|
70
|
+
action: (N, H, A) -- unnormalized
|
|
71
|
+
Returns:
|
|
72
|
+
(N,) tensor.
|
|
73
|
+
"""
|
|
74
|
+
a = critic.action_embed(action, target=False)
|
|
75
|
+
sa = torch.cat([state_embed, a], dim=-1)
|
|
76
|
+
q1 = critic.q1_head(sa).squeeze(-1)
|
|
77
|
+
q2 = critic.q2_head(sa).squeeze(-1)
|
|
78
|
+
return torch.minimum(q1, q2)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# ---------------------------------------------------------------------------
|
|
82
|
+
# Stage I.a: proposal — sample K chunks per state from the diffusion policy.
|
|
83
|
+
# ---------------------------------------------------------------------------
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@torch.no_grad()
|
|
87
|
+
def propose_actions(
|
|
88
|
+
proposer,
|
|
89
|
+
obs_dict: Dict[str, torch.Tensor],
|
|
90
|
+
K: int,
|
|
91
|
+
*,
|
|
92
|
+
inference_steps: Optional[int] = None,
|
|
93
|
+
global_cond_override: Optional[torch.Tensor] = None,
|
|
94
|
+
) -> torch.Tensor:
|
|
95
|
+
"""Sample K UNNORMALIZED action chunks per state.
|
|
96
|
+
|
|
97
|
+
Implements Equation 4.1's "candidate set"
|
|
98
|
+
:math:`\\mathcal{A}_{\\pi_\\varphi, K}(s)` for diffusion policies.
|
|
99
|
+
|
|
100
|
+
Performance note: the visual encoder (typically a frozen ViT-L)
|
|
101
|
+
is the dominant cost of this call. Naively expanding the obs to
|
|
102
|
+
``(B*K, ...)`` before the encoder runs the (heavy, O(image)) ViT
|
|
103
|
+
K times on each unique state. We instead:
|
|
104
|
+
|
|
105
|
+
1. encode the B UNIQUE obs once -> ``global_cond_B: (B, D)``
|
|
106
|
+
2. repeat-interleave the encoded vector to ``(B*K, D)``
|
|
107
|
+
3. run the (lightweight, O(traj)) U-Net at batch B*K with K
|
|
108
|
+
independent noise draws
|
|
109
|
+
|
|
110
|
+
This is mathematically identical to the naive path because the
|
|
111
|
+
encoder is row-independent, but cuts encoder cost K-fold (32x
|
|
112
|
+
at the paper's default K=32). The U-Net cost is unchanged
|
|
113
|
+
because each candidate still gets its own noise sample at every
|
|
114
|
+
DDIM step.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
proposer: a ``DiffusionUnetTimmPolicy``-shaped object exposing
|
|
118
|
+
``normalizer``, ``obs_encoder``, ``action_horizon``,
|
|
119
|
+
``action_dim``, ``device``, ``dtype``,
|
|
120
|
+
``num_inference_steps``, and ``conditional_sample`` /
|
|
121
|
+
``noise_scheduler`` / ``model``. The on-policy snapshot
|
|
122
|
+
(e.g. EMA model) the relabel pass uses.
|
|
123
|
+
obs_dict: ``(B, ...)`` per key, raw (unnormalized) obs.
|
|
124
|
+
K: number of chunks per state.
|
|
125
|
+
inference_steps: temporary override of
|
|
126
|
+
``proposer.num_inference_steps`` for the duration of this
|
|
127
|
+
call. Useful to run cheaper sampling during relabel
|
|
128
|
+
(e.g. 5 DDIM steps) than the policy uses at deployment
|
|
129
|
+
(e.g. 16). Mutating the attribute is safe because we
|
|
130
|
+
ALWAYS restore the original in a try/finally and we hold
|
|
131
|
+
``torch.no_grad`` so no autograd state is tied to the
|
|
132
|
+
attribute. Set to ``None`` to use ``proposer.num_inference_steps``.
|
|
133
|
+
global_cond_override: optional precomputed ``(B, D_cond)``
|
|
134
|
+
tensor produced by the SAME ``proposer.obs_encoder`` for
|
|
135
|
+
the SAME ``obs_dict``. When provided, we skip both the
|
|
136
|
+
normalize and obs_encoder forwards entirely. Intended for
|
|
137
|
+
workflows that pre-compute and cache the encoder output
|
|
138
|
+
once for all dataset frames (the proposer's encoder is
|
|
139
|
+
frozen when ``training.freeze_encoder=True``, so its
|
|
140
|
+
output is invariant across relabel cycles). Must be on
|
|
141
|
+
the same device as the U-Net; dtype is preserved.
|
|
142
|
+
|
|
143
|
+
Returns:
|
|
144
|
+
``(B, K, H, A)`` tensor on ``proposer.device``, UNNORMALIZED
|
|
145
|
+
(matches the contract of ``policy.predict_action``).
|
|
146
|
+
"""
|
|
147
|
+
if K <= 0:
|
|
148
|
+
raise ValueError(f"K must be >= 1, got {K}")
|
|
149
|
+
|
|
150
|
+
B = next(iter(obs_dict.values())).shape[0]
|
|
151
|
+
|
|
152
|
+
if global_cond_override is not None:
|
|
153
|
+
# Cache hit: skip normalize + obs_encoder. Validate shape.
|
|
154
|
+
if global_cond_override.shape[0] != B:
|
|
155
|
+
raise ValueError(
|
|
156
|
+
f"global_cond_override has B="
|
|
157
|
+
f"{global_cond_override.shape[0]} but obs_dict has B={B}"
|
|
158
|
+
)
|
|
159
|
+
global_cond_B = global_cond_override
|
|
160
|
+
else:
|
|
161
|
+
nobs = proposer.normalizer.normalize(obs_dict)
|
|
162
|
+
# Encode once on the B unique obs (NOT B*K) — the K candidates
|
|
163
|
+
# per state share identical visual conditioning; only the U-Net
|
|
164
|
+
# noise differs across replicas. Saves K-fold compute on the
|
|
165
|
+
# visual encoder, which is the dominant relabel cost.
|
|
166
|
+
global_cond_B = proposer.obs_encoder(nobs) # (B, D_cond)
|
|
167
|
+
if K > 1:
|
|
168
|
+
global_cond = global_cond_B.repeat_interleave(K, dim=0)
|
|
169
|
+
else:
|
|
170
|
+
global_cond = global_cond_B
|
|
171
|
+
|
|
172
|
+
saved_steps = proposer.num_inference_steps
|
|
173
|
+
if inference_steps is not None:
|
|
174
|
+
proposer.num_inference_steps = int(inference_steps)
|
|
175
|
+
try:
|
|
176
|
+
cond_data = torch.zeros(
|
|
177
|
+
size=(B * K, proposer.action_horizon, proposer.action_dim),
|
|
178
|
+
device=proposer.device, dtype=proposer.dtype,
|
|
179
|
+
)
|
|
180
|
+
cond_mask = torch.zeros_like(cond_data, dtype=torch.bool)
|
|
181
|
+
nsample = proposer.conditional_sample(
|
|
182
|
+
condition_data=cond_data,
|
|
183
|
+
condition_mask=cond_mask,
|
|
184
|
+
local_cond=None,
|
|
185
|
+
global_cond=global_cond,
|
|
186
|
+
)
|
|
187
|
+
finally:
|
|
188
|
+
proposer.num_inference_steps = saved_steps
|
|
189
|
+
|
|
190
|
+
actions = proposer.normalizer["action"].unnormalize(nsample)
|
|
191
|
+
# (B*K, H, A) -> (B, K, H, A) without copying when K=1.
|
|
192
|
+
actions = actions.view(B, K, proposer.action_horizon, proposer.action_dim)
|
|
193
|
+
return actions
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
# ---------------------------------------------------------------------------
|
|
197
|
+
# Stage I.b: global optimization — rerank K candidates, keep top-m.
|
|
198
|
+
# ---------------------------------------------------------------------------
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
@torch.no_grad()
|
|
202
|
+
def global_optimize(
|
|
203
|
+
critic,
|
|
204
|
+
state_embed_K: torch.Tensor,
|
|
205
|
+
candidates: torch.Tensor,
|
|
206
|
+
m: int,
|
|
207
|
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
208
|
+
"""Equation 4.1: rerank K candidates per state and keep top-m.
|
|
209
|
+
|
|
210
|
+
Args:
|
|
211
|
+
critic: IQL critic (provides ``action_embed``, ``q1_head``,
|
|
212
|
+
``q2_head``).
|
|
213
|
+
state_embed_K: (B*K, D) state embedding, one row per
|
|
214
|
+
candidate. Caller must already have repeated the per-state
|
|
215
|
+
embedding K times (so the critic doesn't re-encode obs per
|
|
216
|
+
candidate).
|
|
217
|
+
candidates: (B, K, H, A) UNNORMALIZED chunks.
|
|
218
|
+
m: number of top candidates to retain. ``m <= K`` enforced.
|
|
219
|
+
|
|
220
|
+
Returns:
|
|
221
|
+
``(top_actions, top_q)`` where
|
|
222
|
+
top_actions: (B, m, H, A)
|
|
223
|
+
top_q: (B, m) -- min(Q1, Q2) of each kept chunk.
|
|
224
|
+
"""
|
|
225
|
+
B, K, H, A = candidates.shape
|
|
226
|
+
if m <= 0:
|
|
227
|
+
raise ValueError(f"m must be >= 1, got {m}")
|
|
228
|
+
if m > K:
|
|
229
|
+
raise ValueError(f"m ({m}) > K ({K})")
|
|
230
|
+
|
|
231
|
+
flat = candidates.reshape(B * K, H, A)
|
|
232
|
+
q_min = q_min_from_state_embed(critic, state_embed_K, flat)
|
|
233
|
+
q_min = q_min.view(B, K)
|
|
234
|
+
|
|
235
|
+
# ``torch.topk`` returns the m highest values and their indices
|
|
236
|
+
# along the K dim. The sort is stable enough for our purposes
|
|
237
|
+
# (ties are vanishingly unlikely on float32 Q).
|
|
238
|
+
top_q, top_idx = q_min.topk(k=m, dim=1, largest=True, sorted=True)
|
|
239
|
+
# Gather the corresponding chunks. expand+gather avoids the
|
|
240
|
+
# advanced-indexing copy.
|
|
241
|
+
idx_exp = top_idx.view(B, m, 1, 1).expand(B, m, H, A)
|
|
242
|
+
top_actions = candidates.gather(dim=1, index=idx_exp).contiguous()
|
|
243
|
+
return top_actions, top_q
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
# ---------------------------------------------------------------------------
|
|
247
|
+
# Stage I.c: local optimization — T grad ascent steps with revert-on-no-improve.
|
|
248
|
+
# ---------------------------------------------------------------------------
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def local_optimize(
|
|
252
|
+
critic,
|
|
253
|
+
state_embed: torch.Tensor,
|
|
254
|
+
actions: torch.Tensor,
|
|
255
|
+
*,
|
|
256
|
+
T: int,
|
|
257
|
+
alpha: float,
|
|
258
|
+
) -> Tuple[torch.Tensor, torch.Tensor, Dict[str, float]]:
|
|
259
|
+
"""Algorithm 1: T steps of gradient ascent on Q w.r.t. action.
|
|
260
|
+
|
|
261
|
+
For each (state, candidate) pair we do up to T steps of
|
|
262
|
+
``a_{j+1} = a_j + alpha * grad_a Q(s, a_j)``. After each step we
|
|
263
|
+
re-evaluate Q; if it did NOT improve, we REVERT to the previous
|
|
264
|
+
action and stop early for that candidate (matching the
|
|
265
|
+
"revert-on-no-improve" rule in Algorithm 1 lines 6-9 -- "we only
|
|
266
|
+
execute a local update if it increases the Q-value", Section 4.3).
|
|
267
|
+
|
|
268
|
+
Crucially: we do NOT re-run the obs encoder during this loop. The
|
|
269
|
+
state embedding is computed once by the caller and passed in,
|
|
270
|
+
while ``critic.action_embed`` (a tiny MLP over the flattened
|
|
271
|
+
chunk) runs each step.
|
|
272
|
+
|
|
273
|
+
Args:
|
|
274
|
+
critic: frozen IQL critic.
|
|
275
|
+
state_embed: (N, D) -- one state-embed per candidate.
|
|
276
|
+
Caller is responsible for replicating per-state embeds to
|
|
277
|
+
match the candidate count.
|
|
278
|
+
actions: (N, H, A) UNNORMALIZED candidates. Returned tensor
|
|
279
|
+
has the same shape; gradient flow does NOT escape this
|
|
280
|
+
function.
|
|
281
|
+
T: max number of gradient steps per candidate.
|
|
282
|
+
alpha: gradient step size in unnormalized action units. The
|
|
283
|
+
paper uses 3e-4 for state-based tasks; image-based PA-RL
|
|
284
|
+
uses the same default.
|
|
285
|
+
|
|
286
|
+
Returns:
|
|
287
|
+
``(actions_opt, q_after, info)`` where
|
|
288
|
+
actions_opt: (N, H, A)
|
|
289
|
+
q_after: (N,) -- final ``min(Q1, Q2)``.
|
|
290
|
+
info: aggregate diagnostics (q_before mean, q_after mean,
|
|
291
|
+
q_gain mean, frac_local_reverted, action_l2_change).
|
|
292
|
+
"""
|
|
293
|
+
if actions.dim() != 3:
|
|
294
|
+
raise ValueError(
|
|
295
|
+
f"local_optimize expects (N, H, A) actions; got {actions.shape}")
|
|
296
|
+
N = actions.shape[0]
|
|
297
|
+
info: Dict[str, float] = {
|
|
298
|
+
"n_candidates": int(N),
|
|
299
|
+
"T": int(T),
|
|
300
|
+
"alpha": float(alpha),
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
if N == 0 or T <= 0 or alpha <= 0.0:
|
|
304
|
+
with torch.no_grad():
|
|
305
|
+
q_now = q_min_from_state_embed(critic, state_embed, actions)
|
|
306
|
+
info.update({
|
|
307
|
+
"q_before_mean": float(q_now.mean().item()),
|
|
308
|
+
"q_after_mean": float(q_now.mean().item()),
|
|
309
|
+
"q_gain_mean": 0.0,
|
|
310
|
+
"frac_local_reverted": 0.0,
|
|
311
|
+
"action_l2_change_mean": 0.0,
|
|
312
|
+
})
|
|
313
|
+
return actions.detach(), q_now.detach(), info
|
|
314
|
+
|
|
315
|
+
# Gradient ascent. We maintain a "current best" copy and only
|
|
316
|
+
# replace it on a Q increase; this is the exact rule in Algorithm 1
|
|
317
|
+
# which the paper treats as the safe knob ("not guaranteed to
|
|
318
|
+
# improve... we only execute a local update if it increases the
|
|
319
|
+
# Q-value", Sec 4.3).
|
|
320
|
+
actions_curr = actions.detach().clone()
|
|
321
|
+
with torch.no_grad():
|
|
322
|
+
q_before = q_min_from_state_embed(critic, state_embed, actions_curr)
|
|
323
|
+
q_curr = q_before.clone()
|
|
324
|
+
|
|
325
|
+
# Track which candidates are still "live". Once a candidate
|
|
326
|
+
# reverts, we freeze it for the rest of the loop (Algorithm 1's
|
|
327
|
+
# explicit `Break`).
|
|
328
|
+
alive = torch.ones(N, dtype=torch.bool, device=actions.device)
|
|
329
|
+
n_steps_used = torch.zeros(N, dtype=torch.long, device=actions.device)
|
|
330
|
+
|
|
331
|
+
for step_idx in range(int(T)):
|
|
332
|
+
if not alive.any():
|
|
333
|
+
break
|
|
334
|
+
# Build a candidate tensor with grads only on the live rows;
|
|
335
|
+
# autograd is happy if the dead rows simply don't contribute
|
|
336
|
+
# to the loss we sum over.
|
|
337
|
+
with torch.enable_grad():
|
|
338
|
+
a_var = actions_curr.detach().clone().requires_grad_(True)
|
|
339
|
+
q_now = q_min_from_state_embed(critic, state_embed, a_var)
|
|
340
|
+
# Sum only the live rows so we don't compute dead-row grads.
|
|
341
|
+
(q_now * alive.float()).sum().backward()
|
|
342
|
+
grad = a_var.grad
|
|
343
|
+
# Per-candidate proposed update. Dead rows get zeroed in-place
|
|
344
|
+
# so they don't contaminate the next iteration.
|
|
345
|
+
proposed = actions_curr + float(alpha) * grad.detach()
|
|
346
|
+
# Re-eval Q at the proposed actions. ``no_grad`` is fine; we
|
|
347
|
+
# just need the scalar comparison.
|
|
348
|
+
with torch.no_grad():
|
|
349
|
+
q_proposed = q_min_from_state_embed(critic, state_embed, proposed)
|
|
350
|
+
improved = (q_proposed > q_curr) & alive
|
|
351
|
+
# Where improved: accept; else: revert (keep actions_curr,
|
|
352
|
+
# q_curr) and mark dead for subsequent steps.
|
|
353
|
+
if improved.any():
|
|
354
|
+
improved_exp = improved.view(N, 1, 1)
|
|
355
|
+
actions_curr = torch.where(
|
|
356
|
+
improved_exp, proposed, actions_curr,
|
|
357
|
+
)
|
|
358
|
+
q_curr = torch.where(improved, q_proposed, q_curr)
|
|
359
|
+
n_steps_used = torch.where(
|
|
360
|
+
improved, n_steps_used + 1, n_steps_used,
|
|
361
|
+
)
|
|
362
|
+
# Mark non-improved (and previously-alive) rows dead.
|
|
363
|
+
alive = alive & improved
|
|
364
|
+
|
|
365
|
+
# Final stats.
|
|
366
|
+
with torch.no_grad():
|
|
367
|
+
delta = (actions_curr - actions).flatten(start_dim=1)
|
|
368
|
+
l2 = delta.norm(dim=-1)
|
|
369
|
+
# A candidate is "reverted" if it didn't improve at step 0,
|
|
370
|
+
# i.e. it took zero accepted steps and Q is unchanged.
|
|
371
|
+
reverted = (n_steps_used == 0)
|
|
372
|
+
info.update({
|
|
373
|
+
"q_before_mean": float(q_before.mean().item()),
|
|
374
|
+
"q_after_mean": float(q_curr.mean().item()),
|
|
375
|
+
"q_gain_mean": float((q_curr - q_before).mean().item()),
|
|
376
|
+
"frac_local_reverted": float(reverted.float().mean().item()),
|
|
377
|
+
"action_l2_change_mean": float(l2.mean().item()),
|
|
378
|
+
"n_steps_mean": float(n_steps_used.float().mean().item()),
|
|
379
|
+
"n_steps_max": int(n_steps_used.max().item()),
|
|
380
|
+
})
|
|
381
|
+
return actions_curr.detach(), q_curr.detach(), info
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
# ---------------------------------------------------------------------------
|
|
385
|
+
# Stage II: pick — argmax or softmax(Q/tau) over the optimized set.
|
|
386
|
+
# ---------------------------------------------------------------------------
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
@dataclass
|
|
390
|
+
class PickResult:
|
|
391
|
+
"""Per-state pick output from ``pick_optimized``."""
|
|
392
|
+
action: torch.Tensor # (B, H, A) -- the chosen (and possibly
|
|
393
|
+
# dataset) action for each state.
|
|
394
|
+
chosen_q: torch.Tensor # (B,) -- Q value at the chosen action.
|
|
395
|
+
chose_dataset: torch.Tensor # (B,) bool -- True iff the dataset
|
|
396
|
+
# action was selected for this state.
|
|
397
|
+
info: Dict[str, Any]
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def pick_optimized(
|
|
401
|
+
critic,
|
|
402
|
+
state_embed: torch.Tensor,
|
|
403
|
+
optimized_actions: torch.Tensor,
|
|
404
|
+
optimized_q: torch.Tensor,
|
|
405
|
+
*,
|
|
406
|
+
mode: str = "argmax",
|
|
407
|
+
softmax_temperature: float = 1.0,
|
|
408
|
+
dataset_action: Optional[torch.Tensor] = None,
|
|
409
|
+
generator: Optional[torch.Generator] = None,
|
|
410
|
+
) -> PickResult:
|
|
411
|
+
"""Equation 4.3: select ONE action per state from the optimized set.
|
|
412
|
+
|
|
413
|
+
Args:
|
|
414
|
+
critic: frozen IQL critic. Used only when ``dataset_action`` is
|
|
415
|
+
provided, to compute its Q value and merge it into the
|
|
416
|
+
candidate set.
|
|
417
|
+
state_embed: (B, D) one row per state. NOT replicated.
|
|
418
|
+
optimized_actions: (B, m, H, A) the post-local-opt actions.
|
|
419
|
+
optimized_q: (B, m) their min(Q1, Q2) values.
|
|
420
|
+
mode: ``"argmax"`` returns ``argmax_a Q(s, a)``;
|
|
421
|
+
``"softmax"`` samples a single action per state from
|
|
422
|
+
``Categorical(logits=Q / tau)``.
|
|
423
|
+
softmax_temperature: tau in the softmax mode. Smaller -> closer
|
|
424
|
+
to argmax; larger -> closer to uniform.
|
|
425
|
+
dataset_action: optional (B, H, A) dataset action chunk to
|
|
426
|
+
include in the candidate set ("we also find that
|
|
427
|
+
additionally including the action a appearing at a given
|
|
428
|
+
state in the dataset into action optimization can sometimes
|
|
429
|
+
be helpful." -- Sec 4.3, Implementation details). If
|
|
430
|
+
provided we evaluate its Q against ``state_embed`` and
|
|
431
|
+
extend the candidate set to (B, m+1, ...).
|
|
432
|
+
generator: optional torch.Generator for reproducible softmax
|
|
433
|
+
sampling.
|
|
434
|
+
|
|
435
|
+
Returns:
|
|
436
|
+
``PickResult``: ``action`` (B, H, A), ``chosen_q`` (B,),
|
|
437
|
+
``chose_dataset`` (B,) bool, ``info`` dict.
|
|
438
|
+
"""
|
|
439
|
+
B, m, H, A = optimized_actions.shape
|
|
440
|
+
if state_embed.shape[0] != B:
|
|
441
|
+
raise ValueError(
|
|
442
|
+
f"state_embed batch ({state_embed.shape[0]}) != "
|
|
443
|
+
f"optimized_actions batch ({B})"
|
|
444
|
+
)
|
|
445
|
+
|
|
446
|
+
# Optionally extend the candidate set with the dataset action.
|
|
447
|
+
if dataset_action is not None:
|
|
448
|
+
if dataset_action.shape != (B, H, A):
|
|
449
|
+
raise ValueError(
|
|
450
|
+
f"dataset_action shape {tuple(dataset_action.shape)} "
|
|
451
|
+
f"!= expected ({B}, {H}, {A})"
|
|
452
|
+
)
|
|
453
|
+
with torch.no_grad():
|
|
454
|
+
q_data = q_min_from_state_embed(
|
|
455
|
+
critic, state_embed, dataset_action,
|
|
456
|
+
) # (B,)
|
|
457
|
+
all_actions = torch.cat(
|
|
458
|
+
[optimized_actions, dataset_action.unsqueeze(1)], dim=1,
|
|
459
|
+
) # (B, m+1, H, A)
|
|
460
|
+
all_q = torch.cat([optimized_q, q_data.unsqueeze(1)], dim=1)
|
|
461
|
+
dataset_idx = m # index of the appended dataset action
|
|
462
|
+
else:
|
|
463
|
+
all_actions = optimized_actions
|
|
464
|
+
all_q = optimized_q
|
|
465
|
+
dataset_idx = None
|
|
466
|
+
|
|
467
|
+
M = all_actions.shape[1]
|
|
468
|
+
|
|
469
|
+
if mode == "argmax":
|
|
470
|
+
chosen = all_q.argmax(dim=1) # (B,)
|
|
471
|
+
elif mode == "softmax":
|
|
472
|
+
if softmax_temperature <= 0:
|
|
473
|
+
raise ValueError(
|
|
474
|
+
f"softmax_temperature must be > 0, got {softmax_temperature}"
|
|
475
|
+
)
|
|
476
|
+
logits = all_q / float(softmax_temperature)
|
|
477
|
+
probs = torch.softmax(logits, dim=1)
|
|
478
|
+
# ``torch.multinomial`` doesn't accept a generator on every
|
|
479
|
+
# backend; fall back to a manual gumbel-max if generator is
|
|
480
|
+
# provided to keep determinism.
|
|
481
|
+
if generator is None:
|
|
482
|
+
chosen = torch.multinomial(probs, num_samples=1).squeeze(1)
|
|
483
|
+
else:
|
|
484
|
+
gumbel = -torch.log(
|
|
485
|
+
-torch.log(
|
|
486
|
+
torch.rand(
|
|
487
|
+
probs.shape, device=probs.device, generator=generator,
|
|
488
|
+
).clamp_min(1e-12)
|
|
489
|
+
).clamp_min(1e-12)
|
|
490
|
+
)
|
|
491
|
+
chosen = (logits + gumbel).argmax(dim=1)
|
|
492
|
+
else:
|
|
493
|
+
raise ValueError(
|
|
494
|
+
f"unknown pick mode {mode!r}; expected 'argmax' or 'softmax'"
|
|
495
|
+
)
|
|
496
|
+
|
|
497
|
+
idx_exp = chosen.view(B, 1, 1, 1).expand(B, 1, H, A)
|
|
498
|
+
action = all_actions.gather(dim=1, index=idx_exp).squeeze(1)
|
|
499
|
+
chosen_q = all_q.gather(dim=1, index=chosen.view(B, 1)).squeeze(1)
|
|
500
|
+
|
|
501
|
+
if dataset_idx is None:
|
|
502
|
+
chose_dataset = torch.zeros(B, dtype=torch.bool, device=action.device)
|
|
503
|
+
else:
|
|
504
|
+
chose_dataset = chosen == dataset_idx
|
|
505
|
+
|
|
506
|
+
info: Dict[str, Any] = {
|
|
507
|
+
"mode": mode,
|
|
508
|
+
"softmax_temperature": float(softmax_temperature),
|
|
509
|
+
"n_candidates": int(M),
|
|
510
|
+
"include_dataset_action": dataset_action is not None,
|
|
511
|
+
"frac_dataset_action_chosen": float(chose_dataset.float().mean().item()),
|
|
512
|
+
"q_chosen_mean": float(chosen_q.mean().item()),
|
|
513
|
+
"q_chosen_std": float(chosen_q.std().item()) if B > 1 else 0.0,
|
|
514
|
+
"q_candidates_std_mean": float(all_q.std(dim=1).mean().item())
|
|
515
|
+
if M > 1 else 0.0,
|
|
516
|
+
}
|
|
517
|
+
return PickResult(
|
|
518
|
+
action=action,
|
|
519
|
+
chosen_q=chosen_q,
|
|
520
|
+
chose_dataset=chose_dataset,
|
|
521
|
+
info=info,
|
|
522
|
+
)
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
# ---------------------------------------------------------------------------
|
|
526
|
+
# End-to-end convenience: propose + global + local + pick.
|
|
527
|
+
# ---------------------------------------------------------------------------
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def optimize_actions(
|
|
531
|
+
*,
|
|
532
|
+
proposer,
|
|
533
|
+
critic,
|
|
534
|
+
obs_dict: Dict[str, torch.Tensor],
|
|
535
|
+
K: int,
|
|
536
|
+
m: int,
|
|
537
|
+
T: int,
|
|
538
|
+
alpha: float,
|
|
539
|
+
pick_mode: str = "argmax",
|
|
540
|
+
softmax_temperature: float = 1.0,
|
|
541
|
+
dataset_action: Optional[torch.Tensor] = None,
|
|
542
|
+
proposer_inference_steps: Optional[int] = None,
|
|
543
|
+
generator: Optional[torch.Generator] = None,
|
|
544
|
+
state_embed_override: Optional[torch.Tensor] = None,
|
|
545
|
+
global_cond_override: Optional[torch.Tensor] = None,
|
|
546
|
+
) -> Tuple[torch.Tensor, Dict[str, Any]]:
|
|
547
|
+
"""Run the full PA-RL action optimization pipeline.
|
|
548
|
+
|
|
549
|
+
Sample K -> rerank top-m -> local Q-grad ascent -> pick (argmax /
|
|
550
|
+
softmax / dataset-augmented). The frozen ``critic`` and the
|
|
551
|
+
``proposer`` (typically a snapshot of the diffusion policy, e.g. the
|
|
552
|
+
EMA model) are used in eval-only mode; this function never touches
|
|
553
|
+
their parameters.
|
|
554
|
+
|
|
555
|
+
Args:
|
|
556
|
+
... (see other args)
|
|
557
|
+
state_embed_override: optional precomputed ``(B, D)`` state
|
|
558
|
+
embedding produced by the SAME ``critic`` for the SAME
|
|
559
|
+
``obs_dict``. When provided, we skip
|
|
560
|
+
``critic.state_embed(obs_dict)`` entirely; intended for
|
|
561
|
+
workflows that pre-compute and cache state embeddings
|
|
562
|
+
once for all dataset frames (the IQL critic's encoder is
|
|
563
|
+
frozen, so its output is invariant across relabel
|
|
564
|
+
cycles). Must be on the same device as the candidate
|
|
565
|
+
actions; dtype is preserved (typically fp16/bf16 if the
|
|
566
|
+
cache was built under autocast).
|
|
567
|
+
|
|
568
|
+
Returns:
|
|
569
|
+
``(action_opt, info)`` where
|
|
570
|
+
action_opt: (B, H, A) UNNORMALIZED chosen action chunk.
|
|
571
|
+
info: dict of stage-wise diagnostics (each prefixed by
|
|
572
|
+
``parl/``) suitable for direct logging to wandb.
|
|
573
|
+
"""
|
|
574
|
+
if K < 1:
|
|
575
|
+
raise ValueError(f"K must be >= 1, got {K}")
|
|
576
|
+
if m < 1:
|
|
577
|
+
raise ValueError(f"m must be >= 1, got {m}")
|
|
578
|
+
if m > K:
|
|
579
|
+
raise ValueError(f"m ({m}) > K ({K})")
|
|
580
|
+
|
|
581
|
+
B = next(iter(obs_dict.values())).shape[0]
|
|
582
|
+
|
|
583
|
+
# Stage I.a: propose K candidates.
|
|
584
|
+
with torch.no_grad():
|
|
585
|
+
candidates = propose_actions(
|
|
586
|
+
proposer, obs_dict, K,
|
|
587
|
+
inference_steps=proposer_inference_steps,
|
|
588
|
+
global_cond_override=global_cond_override,
|
|
589
|
+
) # (B, K, H, A)
|
|
590
|
+
|
|
591
|
+
# Compute state_embed ONCE per state (and reuse it across rerank,
|
|
592
|
+
# local-opt, pick). The critic owns its own normalizer + obs
|
|
593
|
+
# encoder; we just feed it the unnormalized obs dict.
|
|
594
|
+
if state_embed_override is not None:
|
|
595
|
+
# Trust the cache: same critic + same obs -> same embedding.
|
|
596
|
+
# We still validate the leading batch dim to fail loudly on
|
|
597
|
+
# off-by-one indexing bugs in the cache lookup site.
|
|
598
|
+
if state_embed_override.shape[0] != B:
|
|
599
|
+
raise ValueError(
|
|
600
|
+
f"state_embed_override has B={state_embed_override.shape[0]} "
|
|
601
|
+
f"but obs_dict has B={B}"
|
|
602
|
+
)
|
|
603
|
+
state_embed = state_embed_override
|
|
604
|
+
used_cache = True
|
|
605
|
+
else:
|
|
606
|
+
with torch.no_grad():
|
|
607
|
+
state_embed = critic.state_embed(obs_dict, target=False) # (B, D)
|
|
608
|
+
used_cache = False
|
|
609
|
+
|
|
610
|
+
# Stage I.b: global optimization. We need state_embed REPEATED to
|
|
611
|
+
# (B*K, D) so the critic scores each candidate independently.
|
|
612
|
+
state_embed_K = state_embed.repeat_interleave(K, dim=0) # (B*K, D)
|
|
613
|
+
top_actions, top_q = global_optimize(
|
|
614
|
+
critic, state_embed_K, candidates, m,
|
|
615
|
+
) # (B, m, H, A), (B, m)
|
|
616
|
+
|
|
617
|
+
# Stage I.c: local optimization. Same trick: replicate state_embed
|
|
618
|
+
# to (B*m, D).
|
|
619
|
+
state_embed_m = state_embed.repeat_interleave(m, dim=0) # (B*m, D)
|
|
620
|
+
H, A = top_actions.shape[2], top_actions.shape[3]
|
|
621
|
+
flat_actions = top_actions.reshape(B * m, H, A)
|
|
622
|
+
flat_actions_opt, flat_q_after, local_info = local_optimize(
|
|
623
|
+
critic, state_embed_m, flat_actions, T=T, alpha=float(alpha),
|
|
624
|
+
)
|
|
625
|
+
actions_opt = flat_actions_opt.view(B, m, H, A)
|
|
626
|
+
q_after = flat_q_after.view(B, m)
|
|
627
|
+
|
|
628
|
+
# Stage II: pick.
|
|
629
|
+
pick = pick_optimized(
|
|
630
|
+
critic, state_embed, actions_opt, q_after,
|
|
631
|
+
mode=pick_mode,
|
|
632
|
+
softmax_temperature=softmax_temperature,
|
|
633
|
+
dataset_action=dataset_action,
|
|
634
|
+
generator=generator,
|
|
635
|
+
)
|
|
636
|
+
|
|
637
|
+
# Aggregate diagnostics under a parl/ prefix for wandb.
|
|
638
|
+
info: Dict[str, Any] = {
|
|
639
|
+
"parl/K": int(K),
|
|
640
|
+
"parl/m": int(m),
|
|
641
|
+
"parl/T": int(T),
|
|
642
|
+
"parl/alpha": float(alpha),
|
|
643
|
+
"parl/pick_mode": pick_mode,
|
|
644
|
+
"parl/q_global_top1_mean": float(top_q[:, 0].mean().item()),
|
|
645
|
+
"parl/q_global_topm_mean": float(top_q.mean().item()),
|
|
646
|
+
"parl/q_global_spread_mean": float(
|
|
647
|
+
(top_q.max(dim=1).values - top_q.min(dim=1).values).mean().item()
|
|
648
|
+
) if m > 1 else 0.0,
|
|
649
|
+
"parl/q_before_local_mean": local_info["q_before_mean"],
|
|
650
|
+
"parl/q_after_local_mean": local_info["q_after_mean"],
|
|
651
|
+
"parl/q_local_gain_mean": local_info["q_gain_mean"],
|
|
652
|
+
"parl/frac_local_reverted": local_info["frac_local_reverted"],
|
|
653
|
+
"parl/action_l2_change_mean": local_info["action_l2_change_mean"],
|
|
654
|
+
"parl/n_local_steps_mean": local_info.get("n_steps_mean", 0.0),
|
|
655
|
+
"parl/q_chosen_mean": pick.info["q_chosen_mean"],
|
|
656
|
+
"parl/q_chosen_std": pick.info["q_chosen_std"],
|
|
657
|
+
"parl/q_candidates_std_mean": pick.info["q_candidates_std_mean"],
|
|
658
|
+
"parl/frac_dataset_action_chosen": pick.info["frac_dataset_action_chosen"],
|
|
659
|
+
"parl/include_dataset_action": pick.info["include_dataset_action"],
|
|
660
|
+
"parl/B": int(B),
|
|
661
|
+
"parl/state_embed_cache_hit": float(used_cache),
|
|
662
|
+
"parl/global_cond_cache_hit": float(global_cond_override is not None),
|
|
663
|
+
}
|
|
664
|
+
return pick.action, info
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
__all__ = [
|
|
668
|
+
"q_min_from_state_embed",
|
|
669
|
+
"propose_actions",
|
|
670
|
+
"global_optimize",
|
|
671
|
+
"local_optimize",
|
|
672
|
+
"pick_optimized",
|
|
673
|
+
"PickResult",
|
|
674
|
+
"optimize_actions",
|
|
675
|
+
]
|
|
File without changes
|