solarian-diffusion 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. diffusion_policy/__init__.py +0 -0
  2. diffusion_policy/codecs/__init__.py +0 -0
  3. diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
  4. diffusion_policy/common/__init__.py +0 -0
  5. diffusion_policy/common/checkpoint_util.py +64 -0
  6. diffusion_policy/common/json_logger.py +117 -0
  7. diffusion_policy/common/normalize_util.py +251 -0
  8. diffusion_policy/common/pose_repr_util.py +112 -0
  9. diffusion_policy/common/pytorch_util.py +82 -0
  10. diffusion_policy/common/q_critic_util.py +135 -0
  11. diffusion_policy/common/replay_buffer.py +672 -0
  12. diffusion_policy/common/sampler.py +204 -0
  13. diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
  14. diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
  15. diffusion_policy/config/legacy/task/can_image.yaml +64 -0
  16. diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
  17. diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
  18. diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
  19. diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
  20. diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
  21. diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
  22. diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
  23. diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
  24. diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
  25. diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
  26. diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
  27. diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
  28. diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
  29. diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
  30. diffusion_policy/config/legacy/task/square_image.yaml +64 -0
  31. diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
  32. diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
  33. diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
  34. diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
  35. diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
  36. diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
  37. diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
  38. diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
  39. diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
  40. diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
  41. diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
  42. diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
  43. diffusion_policy/config/legacy/task/umi.yaml +45 -0
  44. diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
  45. diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
  46. diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
  47. diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
  48. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
  49. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
  50. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
  51. diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
  52. diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
  53. diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
  54. diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
  55. diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
  56. diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
  57. diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
  58. diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
  59. diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
  60. diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
  61. diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
  62. diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
  63. diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
  64. diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
  65. diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
  66. diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
  67. diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
  68. diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
  69. diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
  70. diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
  71. diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
  72. diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
  73. diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
  74. diffusion_policy/config/rl_annotate.yaml +265 -0
  75. diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
  76. diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
  77. diffusion_policy/config/rl_annotate_v2.yaml +141 -0
  78. diffusion_policy/config/rl_annotate_v3.yaml +165 -0
  79. diffusion_policy/config/task/lift.yaml +89 -0
  80. diffusion_policy/config/task/square.yaml +92 -0
  81. diffusion_policy/config/task/tool_hang.yaml +93 -0
  82. diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
  83. diffusion_policy/config/task/umi.yaml +105 -0
  84. diffusion_policy/config/task/umi_bimanual.yaml +135 -0
  85. diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
  86. diffusion_policy/config/task/umi_image.yaml +36 -0
  87. diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
  88. diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
  89. diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
  90. diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
  91. diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
  92. diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
  93. diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
  94. diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
  95. diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
  96. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
  97. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
  98. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
  99. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
  100. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
  101. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
  102. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
  103. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
  104. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
  105. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
  106. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
  107. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
  108. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
  109. diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
  110. diffusion_policy/config/train_iql_umi.yaml +186 -0
  111. diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
  112. diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
  113. diffusion_policy/dataset/__init__.py +0 -0
  114. diffusion_policy/dataset/base_dataset.py +76 -0
  115. diffusion_policy/dataset/umi_dataset.py +652 -0
  116. diffusion_policy/env_runner/__init__.py +0 -0
  117. diffusion_policy/env_runner/base_image_runner.py +9 -0
  118. diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
  119. diffusion_policy/model/__init__.py +0 -0
  120. diffusion_policy/model/common/__init__.py +0 -0
  121. diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
  122. diffusion_policy/model/common/lr_scheduler.py +46 -0
  123. diffusion_policy/model/common/module_attr_mixin.py +14 -0
  124. diffusion_policy/model/common/normalizer.py +353 -0
  125. diffusion_policy/model/common/tensor_util.py +960 -0
  126. diffusion_policy/model/diffusion/__init__.py +0 -0
  127. diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
  128. diffusion_policy/model/diffusion/conv1d_components.py +46 -0
  129. diffusion_policy/model/diffusion/ema_model.py +88 -0
  130. diffusion_policy/model/diffusion/mask_generator.py +221 -0
  131. diffusion_policy/model/diffusion/positional_embedding.py +17 -0
  132. diffusion_policy/model/vision/__init__.py +0 -0
  133. diffusion_policy/model/vision/crop_randomizer.py +288 -0
  134. diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
  135. diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
  136. diffusion_policy/policy/__init__.py +0 -0
  137. diffusion_policy/policy/base_image_policy.py +27 -0
  138. diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
  139. diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
  140. diffusion_policy/policy/parl_action_optimizer.py +675 -0
  141. diffusion_policy/workspace/__init__.py +0 -0
  142. diffusion_policy/workspace/base_workspace.py +146 -0
  143. diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
  144. solarian_diffusion/__init__.py +32 -0
  145. solarian_diffusion/config.py +31 -0
  146. solarian_diffusion/policy_server.py +316 -0
  147. solarian_diffusion/rtc.py +272 -0
  148. solarian_diffusion/service.py +16 -0
  149. solarian_diffusion/trainer.py +213 -0
  150. solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
  151. solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
  152. solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
  153. solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
  154. solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
  155. solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
  156. solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
  157. umi/__init__.py +0 -0
  158. umi/common/__init__.py +0 -0
  159. umi/common/pose_util.py +115 -0
@@ -0,0 +1,1585 @@
1
+ # Stringified annotations: lets us keep type hints like
2
+ # `self.model: DiffusionUnetImagePolicy` without paying the import cost of
3
+ # `DiffusionUnetImagePolicy` (and friends) at module load. They'll be
4
+ # imported lazily inside `run()` where the runtime values actually matter.
5
+ # Required for the lazy-import block below to be safe across the file.
6
+ from __future__ import annotations
7
+
8
+ if __name__ == "__main__":
9
+ import sys
10
+ import os
11
+ import pathlib
12
+
13
+ ROOT_DIR = str(pathlib.Path(__file__).parent.parent.parent)
14
+ sys.path.append(ROOT_DIR)
15
+ os.chdir(ROOT_DIR)
16
+
17
+ import os
18
+ import hydra
19
+ import torch
20
+ from omegaconf import OmegaConf
21
+ import pathlib
22
+ import copy
23
+ import random
24
+ import numpy as np
25
+ from diffusion_policy.workspace.base_workspace import BaseWorkspace
26
+
27
+ # NOTE: the following imports were previously at module top-level. They
28
+ # are required only by `run()` (training), not by `__init__()` or the
29
+ # inference-time accessors. Importing them at module top adds several
30
+ # seconds to cold-start of the policy server (notably `wandb` and the
31
+ # transitive `accelerate` / dataset / env_runner stack), so we defer
32
+ # them. The `if False:` block is kept to make the migration discoverable
33
+ # to static analyzers / IDEs: it documents what used to be here but is
34
+ # never executed at runtime.
35
+ if False: # pragma: no cover - tooling hint only
36
+ from torch.utils.data import DataLoader # noqa: F401
37
+ import wandb # noqa: F401
38
+ import pickle # noqa: F401
39
+ import tqdm # noqa: F401
40
+ import shutil # noqa: F401
41
+ from diffusion_policy.policy.diffusion_unet_image_policy import DiffusionUnetImagePolicy # noqa: F401
42
+ from diffusion_policy.dataset.base_dataset import BaseImageDataset, BaseDataset # noqa: F401
43
+ from diffusion_policy.env_runner.base_image_runner import BaseImageRunner # noqa: F401
44
+ from diffusion_policy.common.checkpoint_util import TopKCheckpointManager # noqa: F401
45
+ from diffusion_policy.common.json_logger import JsonLogger # noqa: F401
46
+ from diffusion_policy.common.pytorch_util import dict_apply, optimizer_to # noqa: F401
47
+ from diffusion_policy.model.diffusion.ema_model import EMAModel # noqa: F401
48
+ from diffusion_policy.model.common.lr_scheduler import get_scheduler # noqa: F401
49
+ from accelerate import Accelerator # noqa: F401
50
+
51
+ OmegaConf.register_new_resolver("eval", eval, replace=True)
52
+
53
+
54
+ @torch.no_grad()
55
+ def _awr_metrics(policy) -> dict:
56
+ """Pass-through of the policy's per-batch AWR diagnostics.
57
+
58
+ ``DiffusionUnetTimmPolicy.compute_loss`` populates
59
+ ``last_awr_info`` after every forward pass when AWR is active. We
60
+ just forward those scalars to wandb, prefixed under ``awr/``. When
61
+ AWR is off this returns an empty dict (so the metrics never appear
62
+ on the BC training run's wandb panel and don't pollute resumes).
63
+
64
+ Filters to ``awr/*`` keys only so PA-RL diagnostics
65
+ (``parl/*``) don't get double-emitted by both this and
66
+ ``_parl_metrics`` -- they share ``last_awr_info`` since both
67
+ write into it from compute_loss.
68
+ """
69
+ info = getattr(policy, 'last_awr_info', None)
70
+ if not info:
71
+ return {}
72
+ return {k: v for k, v in info.items() if k.startswith('awr/')}
73
+
74
+
75
+ @torch.no_grad()
76
+ def _parl_metrics(policy) -> dict:
77
+ """Pass-through of the policy's per-batch PA-RL diagnostics.
78
+
79
+ ``DiffusionUnetTimmPolicy.compute_loss`` populates
80
+ ``last_awr_info`` with ``parl/*`` keys when the batch carries
81
+ optimized actions; this just forwards them to wandb. Empty dict
82
+ when PA-RL is off so the metrics never appear on AWR / BC runs.
83
+ """
84
+ info = getattr(policy, 'last_awr_info', None)
85
+ if not info:
86
+ return {}
87
+ return {k: v for k, v in info.items() if k.startswith('parl/')}
88
+
89
+
90
+ @torch.no_grad()
91
+ def _bbox_mask_weight_metrics(policy) -> dict:
92
+ """Snapshot of patch_embed.proj weight norms for bbox-mask-channel rgb keys.
93
+
94
+ Returns one block of metrics per rgb key that has a bbox mask routed to
95
+ it (see TimmObsEncoder.bbox_mask_map). For each key we log:
96
+
97
+ * ``bbox_mask/<rgb_key>/w_norm_mask_ch``: L2 norm of the 4th input-
98
+ channel slice of patch_embed.proj. Exactly 0 at training step 0
99
+ (zero-init); growing > 0 proves the optimizer is using the mask.
100
+ * ``bbox_mask/<rgb_key>/w_norm_rgb_mean``: mean L2 norm of the three
101
+ RGB input-channel slices. Stable reference scale to compare against.
102
+ * ``bbox_mask/<rgb_key>/w_ratio``: ratio of the two (mask norm /
103
+ mean rgb norm). A "healthy" trained value is roughly 0.05 - 1.0;
104
+ values close to 0 mean the optimizer declined to use the mask
105
+ (almost certainly because the training data didn't reward it).
106
+
107
+ Returns an empty dict when no bbox is routed via the mask channel
108
+ (e.g. ``bbox_mode: low_dim`` runs), so it's safe to call unconditionally.
109
+ """
110
+ encoder = getattr(policy, 'obs_encoder', None)
111
+ if encoder is None:
112
+ return {}
113
+ mask_map = getattr(encoder, 'bbox_mask_map', None)
114
+ if not mask_map:
115
+ return {}
116
+
117
+ metrics = {}
118
+ for rgb_key in mask_map.keys():
119
+ # nn.ModuleDict doesn't expose .get(); use __contains__ + []
120
+ if rgb_key not in encoder.key_model_map:
121
+ continue
122
+ model = encoder.key_model_map[rgb_key]
123
+ if not hasattr(model, 'patch_embed') \
124
+ or not hasattr(model.patch_embed, 'proj'):
125
+ continue
126
+ weight = model.patch_embed.proj.weight # (out_ch, in_ch=4, k, k)
127
+ if weight.shape[1] < 4:
128
+ continue
129
+ rgb_norms = [weight[:, c].norm().item() for c in range(3)]
130
+ mask_norm = weight[:, 3].norm().item()
131
+ rgb_mean = sum(rgb_norms) / 3.0
132
+ ratio = (mask_norm / rgb_mean) if rgb_mean > 0 else 0.0
133
+ prefix = f'bbox_mask/{rgb_key}'
134
+ metrics[f'{prefix}/w_norm_mask_ch'] = mask_norm
135
+ metrics[f'{prefix}/w_norm_rgb_mean'] = rgb_mean
136
+ metrics[f'{prefix}/w_ratio'] = ratio
137
+ return metrics
138
+
139
+
140
+ class TrainDiffusionUnetImageWorkspace(BaseWorkspace):
141
+ include_keys = ['global_step', 'epoch']
142
+ exclude_keys = tuple()
143
+
144
+ def __init__(self, cfg: OmegaConf, output_dir=None):
145
+ super().__init__(cfg, output_dir=output_dir)
146
+
147
+ # set seed
148
+ seed = cfg.training.seed
149
+ torch.manual_seed(seed)
150
+ np.random.seed(seed)
151
+ random.seed(seed)
152
+
153
+ # configure model
154
+ self.model: DiffusionUnetImagePolicy = hydra.utils.instantiate(cfg.policy)
155
+
156
+ self.ema_model: DiffusionUnetImagePolicy = None
157
+ if cfg.training.use_ema:
158
+ self.ema_model = copy.deepcopy(self.model)
159
+
160
+ # configure training state
161
+ # self.optimizer = hydra.utils.instantiate(
162
+ # cfg.optimizer, params=self.model.parameters())
163
+
164
+ obs_encorder_lr = cfg.optimizer.lr
165
+ if cfg.policy.obs_encoder.pretrained:
166
+ obs_encorder_lr *= 0.1
167
+ print('==> reduce pretrained obs_encorder\'s lr')
168
+ obs_encorder_params = list()
169
+ for param in self.model.obs_encoder.parameters():
170
+ if param.requires_grad:
171
+ obs_encorder_params.append(param)
172
+ print(f'obs_encorder params: {len(obs_encorder_params)}')
173
+ param_groups = [
174
+ {'params': self.model.model.parameters()},
175
+ {'params': obs_encorder_params, 'lr': obs_encorder_lr}
176
+ ]
177
+ # self.optimizer = hydra.utils.instantiate(
178
+ # cfg.optimizer, params=param_groups)
179
+ optimizer_cfg = OmegaConf.to_container(cfg.optimizer, resolve=True)
180
+ optimizer_cfg.pop('_target_')
181
+ self.optimizer = torch.optim.AdamW(
182
+ params=param_groups,
183
+ **optimizer_cfg
184
+ )
185
+
186
+ # configure training state
187
+ self.global_step = 0
188
+ self.epoch = 0
189
+
190
+ # PA-RL: frozen IQL critic is materialized lazily by run() once
191
+ # the device is settled. Kept as None on workspaces that don't
192
+ # opt in, so legacy AWR / BC runs pay nothing.
193
+ self.q_critic = None
194
+
195
+ # PA-RL: optional one-time cache of ``critic.state_embed`` for
196
+ # every dataset frame, built once after the (frozen) critic is
197
+ # loaded and reused across all relabel cycles. Stored as a
198
+ # CPU tensor of shape ``(N_frames, D_state)`` so per-batch
199
+ # lookup is a cheap index_select; ``valid`` flags rows whose
200
+ # anchor_idx couldn't be resolved during the build pass.
201
+ # ``None`` on workspaces that don't opt in (``parl.cache_state_embed=false``).
202
+ # Both fields are ``Optional[torch.Tensor]`` once populated by run().
203
+ self._parl_state_embed_cache = None
204
+ self._parl_state_embed_valid = None
205
+
206
+ # PA-RL: optional one-time cache of the proposer's
207
+ # ``obs_encoder(nobs)`` output for every dataset frame. Built
208
+ # ONCE after the (frozen) proposer encoder is loaded and reused
209
+ # across ALL relabel cycles. Eliminates the second ViT-L
210
+ # forward in propose_actions (the proposer's encoder is the
211
+ # SAME size as the critic's), so combined with the state_embed
212
+ # cache the relabel pass does ZERO ViT-L forwards per iter --
213
+ # all encoder work is amortized into the one-time precompute.
214
+ # Only valid when ``training.freeze_encoder=true``;
215
+ # automatically disabled otherwise.
216
+ self._parl_global_cond_cache = None
217
+ self._parl_global_cond_valid = None
218
+
219
+ # do not save optimizer if resume=False
220
+ if not cfg.training.resume:
221
+ self.exclude_keys = ['optimizer']
222
+
223
+ def _precompute_obs_feature_cache(
224
+ self,
225
+ *,
226
+ train_dataset,
227
+ val_dataset,
228
+ forward_fn,
229
+ cache_label: str,
230
+ n_frames: int,
231
+ device,
232
+ accelerator,
233
+ parl_cfg,
234
+ ):
235
+ """Generic helper: scan train + val once and cache an
236
+ ``(N_frames, D)`` tensor produced by a frozen encoder via
237
+ ``forward_fn(obs_dict_on_device) -> (B, D)``.
238
+
239
+ Both ``_precompute_state_embed_cache`` and
240
+ ``_precompute_global_cond_cache`` use this skeleton; only the
241
+ forward function differs.
242
+
243
+ Notes
244
+ -----
245
+ - Augmentation realism: the dataset's ``__getitem__`` injects
246
+ random start-pose noise (``umi_dataset.py`` ~L538) and the
247
+ obs encoder may apply RandomCrop / ColorJitter. The cache
248
+ therefore stores ONE realization of those random augs per
249
+ frame. This matches the noise budget the encoder's owner
250
+ (IQL critic / BC policy) was trained against (each train
251
+ step also sees a single random aug per frame), so a single
252
+ realization is statistically equivalent to a fresh draw.
253
+ - Validity: any sample whose ``anchor_idx`` is ``-1`` (sampler
254
+ sentinel) leaves its cache row uninitialized; ``valid``
255
+ flags it false and the relabel pass falls back to
256
+ recomputing on the fly for that batch.
257
+
258
+ Args:
259
+ forward_fn: callable taking an ``obs_dict`` of tensors
260
+ on ``device`` and returning a single ``(B, D)``
261
+ tensor on the same device. Wrapped externally in
262
+ ``torch.no_grad`` + ``accelerator.autocast``; do not
263
+ wrap inside the callable.
264
+ cache_label: short human label used in log lines and the
265
+ tqdm description (e.g. ``"state_embed"`` or
266
+ ``"global_cond"``).
267
+
268
+ Returns:
269
+ ``(cache, valid)`` -- pinned CPU ``(N_frames, D)`` tensor
270
+ and ``(N_frames,)`` bool mask.
271
+ """
272
+ from torch.utils.data import DataLoader
273
+ import time as _time
274
+ import tqdm as _tqdm
275
+ from diffusion_policy.common.pytorch_util import dict_apply
276
+
277
+ precompute_bs = int(parl_cfg.get(
278
+ 'precompute_batch_size',
279
+ parl_cfg.get('relabel_batch_size', 16),
280
+ ))
281
+ precompute_workers = int(parl_cfg.get(
282
+ 'precompute_num_workers',
283
+ parl_cfg.get('relabel_num_workers', 4),
284
+ ))
285
+
286
+ # Discover the output dim by running ONE tiny batch. Avoids
287
+ # hardcoding the architecture's hidden dim in the workspace
288
+ # and keeps this method drop-in for any future encoder.
289
+ probe = train_dataset[0]
290
+ probe_obs = dict_apply(
291
+ probe['obs'],
292
+ lambda x: x.unsqueeze(0).to(device, non_blocking=True),
293
+ )
294
+ with torch.no_grad(), accelerator.autocast():
295
+ probe_emb = forward_fn(probe_obs)
296
+ D = int(probe_emb.shape[-1])
297
+ cache_dtype = probe_emb.dtype # bf16 under autocast on A100
298
+ del probe, probe_obs, probe_emb
299
+
300
+ # GPU-resident cache. We considered pinned CPU tensors here
301
+ # but ran into NaN values reading back from pinned bf16
302
+ # tensors on this stack -- pytorch's pin_memory has had
303
+ # subtle bf16 issues across versions. GPU storage fits
304
+ # comfortably (a 200k-frame, D=1024 bf16 cache is ~410 MB)
305
+ # AND eliminates the per-batch H2D copy, so this is the
306
+ # cleaner default. The relabel pass is single-rank in v1,
307
+ # so only the main-process GPU holds the cache.
308
+ cache = torch.zeros(
309
+ (n_frames, D), dtype=cache_dtype, device=device)
310
+ valid = torch.zeros(
311
+ (n_frames,), dtype=torch.bool, device=device)
312
+
313
+ splits = []
314
+ if len(train_dataset) > 0:
315
+ splits.append(('train', train_dataset))
316
+ if val_dataset is not None and len(val_dataset) > 0:
317
+ splits.append(('val', val_dataset))
318
+
319
+ n_seen = 0
320
+ t0_total = _time.perf_counter()
321
+ for split_name, ds in splits:
322
+ # Drop any in-memory action_optimized so the dataset
323
+ # serves the BASE obs (action_optimized doesn't actually
324
+ # change obs but we mirror the relabel pass's defensive
325
+ # save/restore for symmetry).
326
+ saved_actions = ds.action_optimized
327
+ saved_valid = ds.action_optimized_valid
328
+ ds.action_optimized = None
329
+ ds.action_optimized_valid = None
330
+ try:
331
+ loader = DataLoader(
332
+ ds,
333
+ batch_size=precompute_bs,
334
+ shuffle=False,
335
+ num_workers=precompute_workers,
336
+ pin_memory=True,
337
+ persistent_workers=False,
338
+ drop_last=False,
339
+ )
340
+ accelerator.print(
341
+ f"[PA-RL] precompute {cache_label} cache for {split_name}: "
342
+ f"{len(ds)} samples / {len(loader)} batches "
343
+ f"(D={D}, dtype={cache_dtype})"
344
+ )
345
+ t0 = _time.perf_counter()
346
+ pbar = _tqdm.tqdm(
347
+ loader,
348
+ total=len(loader),
349
+ desc=f"PA-RL {cache_label} {split_name}",
350
+ dynamic_ncols=True,
351
+ mininterval=1.0,
352
+ leave=False,
353
+ disable=not accelerator.is_main_process,
354
+ )
355
+ for batch in pbar:
356
+ obs = dict_apply(
357
+ batch['obs'],
358
+ lambda x: x.to(device, non_blocking=True),
359
+ )
360
+ anchor_idx_t = batch['anchor_idx'].to(
361
+ device, non_blocking=True).long()
362
+ with torch.no_grad(), accelerator.autocast():
363
+ emb = forward_fn(obs)
364
+ # Vectorized scatter on GPU: ``cache`` lives on
365
+ # ``device`` so we never round-trip through CPU.
366
+ # ``in_range_t`` filters out the sentinel
367
+ # anchor_idx == -1 rows (sampler emits this when
368
+ # the global frame can't be resolved); those rows
369
+ # stay zero in the cache and ``valid`` flags them
370
+ # false so the relabel pass falls back to fresh
371
+ # compute for that batch.
372
+ in_range_t = (anchor_idx_t >= 0) & (
373
+ anchor_idx_t < n_frames)
374
+ if bool(in_range_t.any().item()):
375
+ valid_anchors_t = anchor_idx_t[in_range_t]
376
+ cache[valid_anchors_t] = emb.detach()[in_range_t]
377
+ valid[valid_anchors_t] = True
378
+ n_seen += int(emb.shape[0])
379
+ pbar.close()
380
+ dt = _time.perf_counter() - t0
381
+ accelerator.print(
382
+ f"[PA-RL] {cache_label} {split_name} done in {dt:.1f}s "
383
+ f"({len(ds) / max(dt, 1e-3):.1f} samples/s)"
384
+ )
385
+ finally:
386
+ ds.action_optimized = saved_actions
387
+ ds.action_optimized_valid = saved_valid
388
+
389
+ n_valid = int(valid.sum().item())
390
+ total_dt = _time.perf_counter() - t0_total
391
+ # Sanity check: any non-finite values in the cache silently
392
+ # propagate to NaN Q-values during relabel and look like the
393
+ # critic has gone insane. Cheap to check once at build time.
394
+ # Only inspect the valid rows (zeros in unused rows are fine).
395
+ if n_valid > 0:
396
+ valid_rows = cache[valid]
397
+ n_bad = int((~torch.isfinite(valid_rows)).any(dim=-1).sum().item())
398
+ if n_bad > 0:
399
+ # Hard-fail rather than silently produce NaN Q-values:
400
+ # there is no recovery from a corrupt feature cache.
401
+ raise RuntimeError(
402
+ f"[PA-RL] {cache_label} cache has {n_bad}/{n_valid} "
403
+ f"valid rows with non-finite values (NaN/Inf). "
404
+ f"This usually means the encoder ran into bf16 "
405
+ f"overflow on this stack. Try setting "
406
+ f"parl.cache_{cache_label}=false to disable the "
407
+ f"cache, or upgrade pytorch."
408
+ )
409
+ accelerator.print(
410
+ f"[PA-RL] {cache_label} cache built in {total_dt:.1f}s: "
411
+ f"{n_valid}/{n_frames} frames valid "
412
+ f"({100.0 * n_valid / max(n_frames, 1):.1f}%), "
413
+ f"{cache.element_size() * cache.nelement() / 1e6:.1f} MB on "
414
+ f"{cache.device}"
415
+ )
416
+ return cache, valid
417
+
418
+ def _precompute_state_embed_cache(
419
+ self,
420
+ *,
421
+ train_dataset,
422
+ val_dataset,
423
+ critic,
424
+ n_frames: int,
425
+ device,
426
+ accelerator,
427
+ parl_cfg,
428
+ ):
429
+ """Precompute and cache ``critic.state_embed`` for every dataset frame.
430
+
431
+ The IQL critic's obs encoder is FROZEN throughout PA-RL fine-
432
+ tuning (``parl.freeze_critic=true``), so its state embedding
433
+ for any given frame is constant across the entire run. Each
434
+ relabel cycle replaces the per-batch
435
+ ``critic.state_embed(obs_dict)`` call (a heavy ViT-L forward)
436
+ with an O(B) ``index_select``.
437
+ """
438
+ critic.eval()
439
+ return self._precompute_obs_feature_cache(
440
+ train_dataset=train_dataset,
441
+ val_dataset=val_dataset,
442
+ forward_fn=lambda obs: critic.state_embed(obs, target=False),
443
+ cache_label="state_embed",
444
+ n_frames=n_frames,
445
+ device=device,
446
+ accelerator=accelerator,
447
+ parl_cfg=parl_cfg,
448
+ )
449
+
450
+ def _precompute_global_cond_cache(
451
+ self,
452
+ *,
453
+ train_dataset,
454
+ val_dataset,
455
+ proposer,
456
+ n_frames: int,
457
+ device,
458
+ accelerator,
459
+ parl_cfg,
460
+ ):
461
+ """Precompute and cache the proposer's ``obs_encoder(nobs)`` output.
462
+
463
+ When ``training.freeze_encoder=True`` the policy's (and EMA
464
+ proposer's) ViT-L visual encoder is frozen for the entire
465
+ PA-RL run. Its output for any given frame is therefore
466
+ constant, so we amortize one full pass over the dataset and
467
+ afterwards every relabel cycle skips the heavy
468
+ ``proposer.obs_encoder(nobs)`` forward inside ``propose_actions``,
469
+ replacing it with an O(B) ``index_select``. This is the same
470
+ optimization as ``_precompute_state_embed_cache`` but for the
471
+ SECOND big ViT-L the relabel pass would otherwise run.
472
+
473
+ The dim D_cond depends on the policy's
474
+ ``timm_obs_encoder.output_shape()`` and is discovered at
475
+ runtime by probing the first sample, so the helper works
476
+ unchanged for any future encoder shape.
477
+ """
478
+ proposer.eval()
479
+
480
+ def _forward(obs):
481
+ nobs = proposer.normalizer.normalize(obs)
482
+ return proposer.obs_encoder(nobs)
483
+
484
+ return self._precompute_obs_feature_cache(
485
+ train_dataset=train_dataset,
486
+ val_dataset=val_dataset,
487
+ forward_fn=_forward,
488
+ cache_label="global_cond",
489
+ n_frames=n_frames,
490
+ device=device,
491
+ accelerator=accelerator,
492
+ parl_cfg=parl_cfg,
493
+ )
494
+
495
+ def _parl_relabel(
496
+ self,
497
+ *,
498
+ train_dataset,
499
+ val_dataset,
500
+ proposer,
501
+ critic,
502
+ n_frames: int,
503
+ action_horizon: int,
504
+ action_dim: int,
505
+ device,
506
+ accelerator,
507
+ parl_cfg,
508
+ state_embed_cache=None,
509
+ state_embed_valid=None,
510
+ global_cond_cache=None,
511
+ global_cond_valid=None,
512
+ ):
513
+ """Run ONE PA-RL relabel pass and return ``(actions, valid, info)``.
514
+
515
+ Iterates the train + val datasets sequentially (no shuffle, no
516
+ DDP sharding -- single-rank pass on accelerator's main
517
+ process), proposes K candidates per state from ``proposer``,
518
+ runs PA-RL action optimization against the frozen ``critic``,
519
+ and scatters the chosen actions into a per-frame
520
+ ``(n_frames, H, A)`` array indexed by the global anchor
521
+ ``anchor_idx`` emitted by ``UmiDataset.__getitem__``.
522
+
523
+ Algorithm 1 + Eq. 4.3 from Sobol Mark et al. 2024.
524
+ Implementation lives in
525
+ ``diffusion_policy.policy.parl_action_optimizer``.
526
+
527
+ Returns:
528
+ actions: (n_frames, H, A) float32; rows where valid=False
529
+ are zeros (defensive default; the dataset emits
530
+ ``has_action_optimized=0`` for those, so the loss
531
+ falls back to dataset action).
532
+ valid: (n_frames,) bool.
533
+ info: dict of aggregated diagnostics under the ``parl/``
534
+ prefix, suitable for ``accelerator.log``.
535
+ """
536
+ from torch.utils.data import DataLoader
537
+ import time as _time
538
+ import tqdm as _tqdm
539
+ from diffusion_policy.policy.parl_action_optimizer import optimize_actions
540
+ from diffusion_policy.common.pytorch_util import dict_apply
541
+
542
+ K = int(parl_cfg.get('k_samples', 32))
543
+ m_top = int(parl_cfg.get('m_top', 10))
544
+ T = int(parl_cfg.get('T_grad_steps', 10))
545
+ alpha = float(parl_cfg.get('alpha_grad', 3.0e-4))
546
+ pick_mode = str(parl_cfg.get('distill_mode', 'argmax'))
547
+ softmax_temp = float(parl_cfg.get('softmax_temperature', 1.0))
548
+ include_data_action = bool(parl_cfg.get('include_dataset_action', True))
549
+ proposer_steps = parl_cfg.get('proposer_inference_steps', None)
550
+ if proposer_steps is not None:
551
+ proposer_steps = int(proposer_steps)
552
+ relabel_batch_size = int(parl_cfg.get('relabel_batch_size', 16))
553
+ relabel_workers = int(parl_cfg.get('relabel_num_workers', 4))
554
+
555
+ # Make sure the proposer / critic are in eval mode and on the
556
+ # right device. We never set requires_grad on either (proposer
557
+ # has frozen visual when freeze_encoder=True; critic is
558
+ # explicitly frozen at load time when freeze_critic=True).
559
+ proposer.eval()
560
+ critic.eval()
561
+
562
+ out_actions = np.zeros((n_frames, action_horizon, action_dim),
563
+ dtype=np.float32)
564
+ out_valid = np.zeros(n_frames, dtype=bool)
565
+
566
+ # Single aggregator across train + val for wandb (we don't
567
+ # need to break out per-split numbers in the diagnostics; the
568
+ # split breakdown is in the per-pass log line below).
569
+ agg = {}
570
+
571
+ splits = []
572
+ if len(train_dataset) > 0:
573
+ splits.append(('train', train_dataset))
574
+ if val_dataset is not None and len(val_dataset) > 0:
575
+ splits.append(('val', val_dataset))
576
+
577
+ n_seen = 0
578
+ t0_total = _time.perf_counter()
579
+ for split_name, ds in splits:
580
+ # Disable any cache that the dataset already has so this
581
+ # pass sees the BASE dataset action (not yesterday's
582
+ # optimized chunk masquerading as the dataset). Restored
583
+ # by the caller after the pass when it calls
584
+ # ``set_action_optimized`` with the new array.
585
+ saved_actions = ds.action_optimized
586
+ saved_valid = ds.action_optimized_valid
587
+ ds.action_optimized = None
588
+ ds.action_optimized_valid = None
589
+ try:
590
+ loader = DataLoader(
591
+ ds,
592
+ batch_size=relabel_batch_size,
593
+ shuffle=False,
594
+ num_workers=relabel_workers,
595
+ pin_memory=True,
596
+ persistent_workers=False,
597
+ drop_last=False,
598
+ )
599
+ accelerator.print(
600
+ f"[PA-RL] relabel {split_name}: "
601
+ f"{len(ds)} samples / {len(loader)} batches "
602
+ f"(K={K}, m={m_top}, T={T}, alpha={alpha:g}, "
603
+ f"DDIM_steps={proposer_steps if proposer_steps is not None else 'policy_default'})"
604
+ )
605
+ t0 = _time.perf_counter()
606
+ # Live tqdm bar over the relabel batches. Postfix shows
607
+ # the running mean Q before / after local optimization
608
+ # so the operator can confirm Q is improving (not just
609
+ # that the loop is moving) without needing to wait for
610
+ # the per-cycle wandb log at the end.
611
+ pbar = _tqdm.tqdm(
612
+ loader,
613
+ total=len(loader),
614
+ desc=f"PA-RL relabel {split_name}",
615
+ dynamic_ncols=True,
616
+ mininterval=1.0,
617
+ leave=False,
618
+ disable=not accelerator.is_main_process,
619
+ )
620
+ # Running Q sums for the postfix; intentionally weighted
621
+ # by batch size so partial batches don't bias the mean.
622
+ q_before_running = 0.0
623
+ q_after_running = 0.0
624
+ q_chosen_running = 0.0
625
+ q_n_running = 0
626
+ for batch_idx, batch in enumerate(pbar):
627
+ obs = dict_apply(
628
+ batch['obs'],
629
+ lambda x: x.to(device, non_blocking=True),
630
+ )
631
+ anchor_idx_t = batch['anchor_idx']
632
+ anchor_idx = anchor_idx_t.cpu().numpy()
633
+ dataset_action = None
634
+ if include_data_action:
635
+ dataset_action = batch['action'].to(
636
+ device, non_blocking=True)
637
+
638
+ # Cache lookup: when the frozen-critic state_embed
639
+ # cache is populated AND every row in this batch
640
+ # has a valid cached embedding, we hand it to
641
+ # optimize_actions and skip the heavy ViT-L
642
+ # forward inside ``critic.state_embed``. If ANY
643
+ # row is invalid (anchor_idx == -1, or never
644
+ # written), we fall back to recomputing on the
645
+ # fly so correctness is preserved at the cost of
646
+ # one extra forward for that batch.
647
+ # Cache lookups for the two big ViT-L outputs:
648
+ # - state_embed_cache -> critic.state_embed(obs)
649
+ # - global_cond_cache -> proposer.obs_encoder(nobs)
650
+ # Both are computed once at run() startup and
651
+ # constant for the rest of training (their owners
652
+ # are frozen). Caches live ON the GPU so per-batch
653
+ # we just do an O(B) GPU-side ``index_select``;
654
+ # no H2D copy. Falls back to recomputing on the
655
+ # fly if any row in this batch has an unresolved
656
+ # ``anchor_idx``.
657
+ ai_long = anchor_idx_t.to(
658
+ device, non_blocking=True).long()
659
+ state_embed_override = None
660
+ if state_embed_cache is not None:
661
+ in_range = (ai_long >= 0) & (
662
+ ai_long < state_embed_cache.shape[0])
663
+ if bool(in_range.all().item()):
664
+ sv = state_embed_valid.index_select(0, ai_long)
665
+ if bool(sv.all().item()):
666
+ state_embed_override = (
667
+ state_embed_cache.index_select(
668
+ 0, ai_long))
669
+ global_cond_override = None
670
+ if global_cond_cache is not None:
671
+ in_range_g = (ai_long >= 0) & (
672
+ ai_long < global_cond_cache.shape[0])
673
+ if bool(in_range_g.all().item()):
674
+ gv = global_cond_valid.index_select(0, ai_long)
675
+ if bool(gv.all().item()):
676
+ global_cond_override = (
677
+ global_cond_cache.index_select(
678
+ 0, ai_long))
679
+
680
+ # Honor the accelerator's mixed_precision setting
681
+ # for the relabel pass too (training already does
682
+ # via accelerator.prepare). The proposer's ViT-L
683
+ # and the critic's state_embed dominate runtime
684
+ # and benefit ~3-4x from bf16 on A100. The local
685
+ # gradient ascent on the action runs through the
686
+ # critic's q_heads inside the autocast region;
687
+ # autograd handles dtype promotion to fp32 for
688
+ # the leaf action grad correctly.
689
+ with accelerator.autocast():
690
+ action_opt, info = optimize_actions(
691
+ proposer=proposer,
692
+ critic=critic,
693
+ obs_dict=obs,
694
+ K=K, m=m_top, T=T, alpha=alpha,
695
+ pick_mode=pick_mode,
696
+ softmax_temperature=softmax_temp,
697
+ dataset_action=dataset_action,
698
+ proposer_inference_steps=proposer_steps,
699
+ state_embed_override=state_embed_override,
700
+ global_cond_override=global_cond_override,
701
+ )
702
+ a_np = action_opt.detach().float().cpu().numpy()
703
+ # Vectorized scatter: O(B) Python loops add tens
704
+ # of microseconds per row that don't scale with
705
+ # GPU work, so they punish us at large B. Build a
706
+ # boolean mask once and let numpy do the dispatch.
707
+ in_range = (anchor_idx >= 0) & (anchor_idx < n_frames)
708
+ if in_range.any():
709
+ valid_anchors = anchor_idx[in_range]
710
+ out_actions[valid_anchors] = a_np[in_range]
711
+ out_valid[valid_anchors] = True
712
+ n_seen += int(a_np.shape[0])
713
+ for k_, v_ in info.items():
714
+ if isinstance(v_, (bool,)):
715
+ v_ = float(v_)
716
+ if isinstance(v_, (int, float)):
717
+ agg.setdefault(k_, []).append(float(v_))
718
+
719
+ # Update tqdm postfix every batch from the same
720
+ # ``info`` dict optimize_actions just returned. The
721
+ # running averages here weight each batch by its
722
+ # size so the displayed numbers are unbiased even
723
+ # if the last batch is partial.
724
+ bsz = int(a_np.shape[0])
725
+ q_before_running += float(
726
+ info.get('parl/q_before_local_mean', 0.0)) * bsz
727
+ q_after_running += float(
728
+ info.get('parl/q_after_local_mean', 0.0)) * bsz
729
+ q_chosen_running += float(
730
+ info.get('parl/q_chosen_mean', 0.0)) * bsz
731
+ q_n_running += bsz
732
+ if q_n_running > 0:
733
+ pbar.set_postfix({
734
+ 'Q_before': f"{q_before_running / q_n_running:+.3f}",
735
+ 'Q_after': f"{q_after_running / q_n_running:+.3f}",
736
+ 'Q_chosen': f"{q_chosen_running / q_n_running:+.3f}",
737
+ 'rev': f"{float(info.get('parl/frac_local_reverted', 0.0)):.2f}",
738
+ 'data_picked': f"{float(info.get('parl/frac_dataset_action_chosen', 0.0)):.2f}",
739
+ }, refresh=False)
740
+ pbar.close()
741
+ dt = _time.perf_counter() - t0
742
+ accelerator.print(
743
+ f"[PA-RL] relabel {split_name} done in {dt:.1f}s "
744
+ f"({len(ds) / max(dt, 1e-3):.1f} samples/s)"
745
+ )
746
+ finally:
747
+ ds.action_optimized = saved_actions
748
+ ds.action_optimized_valid = saved_valid
749
+
750
+ info_means = {k: float(np.mean(v)) for k, v in agg.items() if v}
751
+ info_means['parl/n_relabeled_frames'] = int(out_valid.sum())
752
+ info_means['parl/n_total_frames'] = int(n_frames)
753
+ info_means['parl/n_samples_seen'] = int(n_seen)
754
+ info_means['parl/relabel_seconds'] = float(
755
+ _time.perf_counter() - t0_total)
756
+ return out_actions, out_valid, info_means
757
+
758
+ def run(self):
759
+ # Lazy imports: these are training-only dependencies. Keeping them
760
+ # out of module-top imports saves several seconds of cold start when
761
+ # the workspace class is loaded purely for inference (e.g. by
762
+ # policy_server.py -> hydra.utils.get_class). The first call to
763
+ # run() pays the import cost; the imports are then cached in
764
+ # sys.modules so subsequent calls are free.
765
+ from torch.utils.data import DataLoader
766
+ from solarian.tracking.accelerate import accelerate_trackers
767
+ import pickle
768
+ import tqdm
769
+ import shutil
770
+ from diffusion_policy.policy.diffusion_unet_image_policy import DiffusionUnetImagePolicy
771
+ from diffusion_policy.dataset.base_dataset import BaseImageDataset, BaseDataset
772
+ from diffusion_policy.env_runner.base_image_runner import BaseImageRunner
773
+ from diffusion_policy.common.checkpoint_util import TopKCheckpointManager
774
+ from diffusion_policy.common.json_logger import JsonLogger
775
+ from diffusion_policy.common.pytorch_util import dict_apply, optimizer_to
776
+ from diffusion_policy.model.diffusion.ema_model import EMAModel
777
+ from diffusion_policy.model.common.lr_scheduler import get_scheduler
778
+ from accelerate import Accelerator, DistributedDataParallelKwargs
779
+
780
+ cfg = copy.deepcopy(self.cfg)
781
+
782
+ # When the visual encoder is frozen (typical for AWR finetuning),
783
+ # its parameters don't receive gradients. The default DDP wrap
784
+ # then fails with "Expected to have finished reduction in the
785
+ # prior iteration..." because DDP's reducer expects every
786
+ # registered parameter to participate in backward. The fix is
787
+ # to construct DDP with find_unused_parameters=True so it
788
+ # tolerates the no-grad params; we gate this on freeze_encoder
789
+ # to avoid the (small) per-step graph-traversal overhead in
790
+ # plain BC training where everything trains.
791
+ ddp_kwargs = DistributedDataParallelKwargs(
792
+ find_unused_parameters=bool(
793
+ cfg.training.get('freeze_encoder', False)
794
+ )
795
+ )
796
+ # logging.backend (wandb, solarian_base, none, or a list) picks where
797
+ # the run goes; tracking drops empty tags and clamps long ones.
798
+ logging_cfg = OmegaConf.to_container(cfg.logging, resolve=True)
799
+ log_with, init_kwargs = accelerate_trackers(logging_cfg)
800
+ accelerator = Accelerator(
801
+ log_with=log_with,
802
+ kwargs_handlers=[ddp_kwargs],
803
+ )
804
+ accelerator.init_trackers(
805
+ project_name=cfg.logging.project,
806
+ config=OmegaConf.to_container(cfg, resolve=True),
807
+ init_kwargs=init_kwargs,
808
+ )
809
+
810
+ # resume training
811
+ if cfg.training.resume:
812
+ # Support custom checkpoint path via config, or use latest from output_dir
813
+ if hasattr(cfg.training, 'checkpoint_path') and cfg.training.checkpoint_path:
814
+ ckpt_path = pathlib.Path(cfg.training.checkpoint_path)
815
+ if ckpt_path.is_file():
816
+ accelerator.print(f"Resuming from checkpoint {ckpt_path}")
817
+ self.load_checkpoint(path=ckpt_path)
818
+ else:
819
+ accelerator.print(f"Warning: Checkpoint path {ckpt_path} not found, skipping resume")
820
+ else:
821
+ lastest_ckpt_path = self.get_checkpoint_path()
822
+ if lastest_ckpt_path.is_file():
823
+ accelerator.print(f"Resuming from checkpoint {lastest_ckpt_path}")
824
+ self.load_checkpoint(path=lastest_ckpt_path)
825
+ elif (
826
+ hasattr(cfg.training, 'init_from_ckpt')
827
+ and cfg.training.init_from_ckpt
828
+ ):
829
+ # Fresh run that *initializes* model weights from another
830
+ # checkpoint without resuming optimizer / scheduler state.
831
+ # Used for AWR finetuning of a pre-trained BC policy:
832
+ # we want the BC weights as a starting point but a fresh
833
+ # optimizer (so the LR schedule restarts cleanly), fresh
834
+ # global_step / epoch counters, and (importantly) the *new*
835
+ # cfg's policy hyperparameters in self.model -- including
836
+ # awr_beta etc.
837
+ init_path = pathlib.Path(cfg.training.init_from_ckpt)
838
+ if init_path.is_file():
839
+ accelerator.print(
840
+ f"Initializing model weights from {init_path} "
841
+ f"(optimizer + step counters NOT loaded)"
842
+ )
843
+ # Skip optimizer state -- the new run uses a fresh
844
+ # optimizer with possibly-different LR. include_keys=()
845
+ # skips global_step / epoch pickles too. The AWR
846
+ # config knobs are plain Python attributes (set in
847
+ # __init__ from the new cfg, not stored in state_dict),
848
+ # so a strict load of the BC ckpt's `model` and
849
+ # `ema_model` state_dicts produces no key mismatch.
850
+ self.load_checkpoint(
851
+ path=init_path,
852
+ exclude_keys=['optimizer'],
853
+ include_keys=(),
854
+ )
855
+ # Force a clean restart of the schedule even though
856
+ # global_step/epoch were already 0 from __init__.
857
+ self.global_step = 0
858
+ self.epoch = 0
859
+
860
+ # BC-anchored AWR: snapshot the freshly-loaded BC
861
+ # weights into a frozen reference U-Net on the policy.
862
+ # No-op when ``policy.awr_anchor_lambda <= 0``, so
863
+ # legacy AWR / BC runs pay nothing extra. We do this
864
+ # here (not in policy.__init__) because the policy is
865
+ # built from the new AWR cfg with random weights; the
866
+ # anchor must be the BC ckpt's weights, not the random
867
+ # init.
868
+ init_fn = getattr(self.model, 'init_frozen_ref', None)
869
+ if callable(init_fn):
870
+ accelerator.print(
871
+ "Calling policy.init_frozen_ref() to snapshot "
872
+ "BC weights for anchor loss (no-op when "
873
+ "awr_anchor_lambda <= 0).")
874
+ init_fn()
875
+ # Mirror onto the EMA shadow so its forward pass
876
+ # under sample_every / validation can also access
877
+ # the anchor reference. The deep-copied ref itself
878
+ # is shared by attribute reference -- it's frozen
879
+ # and never updated, so sharing is safe.
880
+ if (
881
+ cfg.training.use_ema
882
+ and getattr(self, 'ema_model', None) is not None
883
+ ):
884
+ object.__setattr__(
885
+ self.ema_model, 'frozen_ref_model',
886
+ getattr(self.model, 'frozen_ref_model', None),
887
+ )
888
+ else:
889
+ accelerator.print(
890
+ f"Warning: init_from_ckpt={init_path} not found; "
891
+ f"training will start from random / pretrained weights."
892
+ )
893
+
894
+ # ============== PA-RL setup ==============
895
+ # Resolve PA-RL config block (may be absent). Fields:
896
+ # parl.enabled bool master switch
897
+ # parl.relabel_every int epochs between relabels
898
+ # parl.k_samples / m_top / T_grad_steps / alpha_grad
899
+ # parl.distill_mode 'argmax' | 'softmax'
900
+ # parl.softmax_temperature float
901
+ # parl.proposer 'ema' | 'main'
902
+ # parl.include_dataset_action bool
903
+ # parl.proposer_inference_steps int (override DDIM steps)
904
+ # parl.relabel_batch_size / num_workers int
905
+ # parl.freeze_critic bool (always True for v1)
906
+ # Q-critic checkpoint path lives at ``training.q_critic_ckpt``
907
+ # for symmetry with ``training.init_from_ckpt`` (BC ckpt).
908
+ parl_cfg_node = None
909
+ try:
910
+ if OmegaConf.select(cfg, 'parl') is not None:
911
+ parl_cfg_node = cfg.parl
912
+ except Exception:
913
+ parl_cfg_node = None
914
+ parl_enabled = bool(
915
+ parl_cfg_node is not None
916
+ and OmegaConf.select(parl_cfg_node, 'enabled', default=False)
917
+ )
918
+ q_ckpt_path = OmegaConf.select(
919
+ cfg, 'training.q_critic_ckpt', default=None)
920
+
921
+ if parl_enabled and q_ckpt_path:
922
+ from diffusion_policy.common.q_critic_util import load_iql_critic
923
+ init_path = pathlib.Path(os.path.expanduser(str(q_ckpt_path)))
924
+ if not init_path.is_file():
925
+ raise FileNotFoundError(
926
+ f"PA-RL: training.q_critic_ckpt={init_path} not found."
927
+ )
928
+ # Load critic to CPU first; we'll move it to device once
929
+ # `accelerator.prepare()` has resolved the actual device
930
+ # below. This also matches policy_server.load_q_critic's
931
+ # contract (CPU -> device after build).
932
+ accelerator.print(
933
+ f"[PA-RL] loading frozen Q critic from {init_path}")
934
+ q_critic, q_cfg = load_iql_critic(
935
+ str(init_path), torch.device('cpu'),
936
+ )
937
+ # Sanity-check action shape vs the diffusion policy.
938
+ policy_h = int(self.model.action_horizon)
939
+ policy_a = int(self.model.action_dim)
940
+ critic_h = int(q_critic.action_horizon)
941
+ critic_a = int(q_critic.action_dim)
942
+ if (critic_h, critic_a) != (policy_h, policy_a):
943
+ raise RuntimeError(
944
+ f"PA-RL: Q-critic action shape ({critic_h}, {critic_a}) "
945
+ f"!= diffusion policy ({policy_h}, {policy_a}); "
946
+ f"the relabel pass would produce nonsense Q values."
947
+ )
948
+ # Freeze critic params (default; the freeze_critic flag is
949
+ # plumbed for forward-compat with future Cal-QL co-training).
950
+ freeze_critic = bool(OmegaConf.select(
951
+ parl_cfg_node, 'freeze_critic', default=True))
952
+ if freeze_critic:
953
+ for p in q_critic.parameters():
954
+ p.requires_grad_(False)
955
+ self.q_critic = q_critic
956
+ accelerator.print(
957
+ f"[PA-RL] q_critic ready (frozen={freeze_critic}); "
958
+ f"action=({critic_h}, {critic_a})"
959
+ )
960
+ elif parl_enabled and not q_ckpt_path:
961
+ accelerator.print(
962
+ "[PA-RL] parl.enabled=true but training.q_critic_ckpt is "
963
+ "unset; PA-RL will be DISABLED (relabel pass skipped, "
964
+ "dataset emits no action_optimized)."
965
+ )
966
+ parl_enabled = False
967
+
968
+ # configure dataset
969
+ dataset: BaseImageDataset
970
+ dataset = hydra.utils.instantiate(cfg.task.dataset)
971
+ assert isinstance(dataset, BaseImageDataset) or isinstance(dataset, BaseDataset)
972
+ train_dataloader = DataLoader(dataset, **cfg.dataloader)
973
+
974
+ # compute normalizer on the main process and save to disk
975
+ normalizer_path = os.path.join(self.output_dir, 'normalizer.pkl')
976
+ if accelerator.is_main_process:
977
+ normalizer = dataset.get_normalizer()
978
+ pickle.dump(normalizer, open(normalizer_path, 'wb'))
979
+
980
+ # load normalizer on all processes
981
+ accelerator.wait_for_everyone()
982
+ normalizer = pickle.load(open(normalizer_path, 'rb'))
983
+
984
+ # configure validation dataset
985
+ val_dataset = dataset.get_validation_dataset()
986
+ val_dataloader = DataLoader(val_dataset, **cfg.val_dataloader)
987
+ print('train dataset:', len(dataset), 'train dataloader:', len(train_dataloader))
988
+ print('val dataset:', len(val_dataset), 'val dataloader:', len(val_dataloader))
989
+
990
+ self.model.set_normalizer(normalizer)
991
+ if cfg.training.use_ema:
992
+ self.ema_model.set_normalizer(normalizer)
993
+
994
+ # Set initial_lr for each param group if resuming (required by LR scheduler)
995
+ if cfg.training.resume and self.global_step > 0:
996
+ for param_group in self.optimizer.param_groups:
997
+ if 'initial_lr' not in param_group:
998
+ param_group['initial_lr'] = param_group['lr']
999
+
1000
+ # configure lr scheduler
1001
+ lr_scheduler = get_scheduler(
1002
+ cfg.training.lr_scheduler,
1003
+ optimizer=self.optimizer,
1004
+ num_warmup_steps=cfg.training.lr_warmup_steps,
1005
+ num_training_steps=(
1006
+ len(train_dataloader) * cfg.training.num_epochs) \
1007
+ // cfg.training.gradient_accumulate_every,
1008
+ # pytorch assumes stepping LRScheduler every epoch
1009
+ # however huggingface diffusers steps it every batch
1010
+ last_epoch=self.global_step-1
1011
+ )
1012
+
1013
+ # configure ema
1014
+ ema: EMAModel = None
1015
+ if cfg.training.use_ema:
1016
+ ema = hydra.utils.instantiate(
1017
+ cfg.ema,
1018
+ model=self.ema_model)
1019
+
1020
+ # configure env
1021
+ env_runner: BaseImageRunner
1022
+ env_runner = hydra.utils.instantiate(
1023
+ cfg.task.env_runner,
1024
+ output_dir=self.output_dir)
1025
+ assert isinstance(env_runner, BaseImageRunner)
1026
+
1027
+ # # configure logging
1028
+ # wandb_run = wandb.init(
1029
+ # dir=str(self.output_dir),
1030
+ # config=OmegaConf.to_container(cfg, resolve=True),
1031
+ # **cfg.logging
1032
+ # )
1033
+ # wandb.config.update(
1034
+ # {
1035
+ # "output_dir": self.output_dir,
1036
+ # }
1037
+ # )
1038
+
1039
+ # configure checkpoint
1040
+ # Build constant fields used in checkpoint filenames from the experiment
1041
+ # config (task name, model type, visual encoder, total batch size,
1042
+ # dataset file name without extension). These are optional in cfg, so
1043
+ # missing values fall back to 'na' to avoid breaking other workspaces
1044
+ # that share this manager.
1045
+ dataset_path = cfg.task.get('dataset_path', '') if hasattr(cfg, 'task') else ''
1046
+ dataset_file_name = os.path.basename(str(dataset_path))
1047
+ # Strip common archive/zarr extensions (e.g. .zarr.zip, .zarr, .zip).
1048
+ while True:
1049
+ stem, ext = os.path.splitext(dataset_file_name)
1050
+ if ext.lower() in ('.zip', '.zarr', '.gz', '.tar'):
1051
+ dataset_file_name = stem
1052
+ else:
1053
+ break
1054
+ if not dataset_file_name:
1055
+ dataset_file_name = 'na'
1056
+
1057
+ total_batch = int(cfg.get('batch_size', 0)) * int(cfg.get('num_gpus', 1))
1058
+
1059
+ static_format_kwargs = {
1060
+ 'task_name': cfg.get('task_name', 'na'),
1061
+ 'model_type': cfg.get('model_type', 'na'),
1062
+ 'visual_encoder': cfg.get('visual_encoder', 'na'),
1063
+ 'total_batch': total_batch,
1064
+ 'dataset_file_name': dataset_file_name,
1065
+ }
1066
+
1067
+ topk_manager = TopKCheckpointManager(
1068
+ save_dir=os.path.join(self.output_dir, 'checkpoints'),
1069
+ static_format_kwargs=static_format_kwargs,
1070
+ **cfg.checkpoint.topk
1071
+ )
1072
+
1073
+ # device transfer
1074
+ # device = torch.device(cfg.training.device)
1075
+ # self.model.to(device)
1076
+ # if self.ema_model is not None:
1077
+ # self.ema_model.to(device)
1078
+ # optimizer_to(self.optimizer, device)
1079
+
1080
+ # accelerator
1081
+ train_dataloader, val_dataloader, self.model, self.optimizer, lr_scheduler = accelerator.prepare(
1082
+ train_dataloader, val_dataloader, self.model, self.optimizer, lr_scheduler
1083
+ )
1084
+ device = self.model.device
1085
+ if self.ema_model is not None:
1086
+ self.ema_model.to(device)
1087
+
1088
+ # PA-RL: move the (already-frozen) Q critic to the same device.
1089
+ # Done AFTER accelerator.prepare so single- and multi-GPU runs
1090
+ # both pick up the right device id without re-reading config.
1091
+ # We never DDP-wrap the critic: it's frozen in v1, used only in
1092
+ # eval(), and the relabel pass runs single-rank on the main
1093
+ # process anyway.
1094
+ if self.q_critic is not None:
1095
+ self.q_critic.to(device)
1096
+ self.q_critic.eval()
1097
+
1098
+ # PA-RL: optional one-time precompute of ``critic.state_embed``
1099
+ # for every dataset frame. Saves the heavy ViT-L state_embed
1100
+ # forward on EVERY relabel cycle thereafter; replaced by an
1101
+ # O(B) ``index_select`` from a pinned CPU tensor and a single
1102
+ # H2D copy. Only runs when ``parl.cache_state_embed=true`` AND
1103
+ # the critic is frozen (the cache is invalid if the critic's
1104
+ # encoder changes mid-run). Single-rank: main process owns
1105
+ # the build; other ranks wait at the barrier and currently
1106
+ # don't share the cache (the relabel pass is also single-rank
1107
+ # in v1, so rank>0 never hits the lookup site).
1108
+ cache_state_embed = (
1109
+ parl_enabled
1110
+ and self.q_critic is not None
1111
+ and bool(OmegaConf.select(
1112
+ cfg, 'parl.cache_state_embed', default=True))
1113
+ and bool(OmegaConf.select(
1114
+ cfg, 'parl.freeze_critic', default=True))
1115
+ )
1116
+ if cache_state_embed and accelerator.is_main_process:
1117
+ cache_n_frames = int(getattr(dataset, 'n_frames_total', 0))
1118
+ if cache_n_frames > 0:
1119
+ cache_t, cache_v = self._precompute_state_embed_cache(
1120
+ train_dataset=dataset,
1121
+ val_dataset=val_dataset,
1122
+ critic=self.q_critic,
1123
+ n_frames=cache_n_frames,
1124
+ device=device,
1125
+ accelerator=accelerator,
1126
+ parl_cfg=cfg.parl,
1127
+ )
1128
+ self._parl_state_embed_cache = cache_t
1129
+ self._parl_state_embed_valid = cache_v
1130
+ accelerator.wait_for_everyone()
1131
+
1132
+ # PA-RL: optional one-time precompute of the proposer's
1133
+ # ``obs_encoder`` output for every dataset frame. Same idea
1134
+ # as the state_embed cache above but for the SECOND big
1135
+ # ViT-L the relabel pass would otherwise run (the proposer's
1136
+ # visual encoder inside ``propose_actions``). Combined, the
1137
+ # two caches eliminate ALL ViT-L forwards from the per-iter
1138
+ # relabel cost; the only remaining heavy compute is the
1139
+ # U-Net's K candidate diffusion samples plus the tiny
1140
+ # action/q-head MLPs.
1141
+ # Gated on (a) the proposer encoder actually being frozen
1142
+ # (``training.freeze_encoder=true``) and (b)
1143
+ # ``parl.cache_global_cond=true`` (default true). The proposer
1144
+ # we cache against is the EMA snapshot when available --
1145
+ # matches the proposer the relabel pass uses; the encoder
1146
+ # weights are identical to the underlying model's because
1147
+ # EMA only steps params that move and the encoder doesn't.
1148
+ cache_global_cond = (
1149
+ parl_enabled
1150
+ and bool(OmegaConf.select(
1151
+ cfg, 'parl.cache_global_cond', default=True))
1152
+ and bool(OmegaConf.select(
1153
+ cfg, 'training.freeze_encoder', default=False))
1154
+ )
1155
+ if cache_global_cond and accelerator.is_main_process:
1156
+ cache_n_frames = int(getattr(dataset, 'n_frames_total', 0))
1157
+ if cache_n_frames > 0:
1158
+ cache_proposer = (
1159
+ self.ema_model if self.ema_model is not None
1160
+ else accelerator.unwrap_model(self.model)
1161
+ )
1162
+ cache_proposer.eval()
1163
+ # Encoder must be in eval mode for deterministic
1164
+ # transforms, in particular to ensure dropout (if
1165
+ # any) is disabled. requires_grad is already false
1166
+ # via the freeze_encoder enforcement at the top of
1167
+ # the training loop, but eval() also matters for
1168
+ # batchnorm / dropout layers a future encoder might
1169
+ # carry.
1170
+ cache_proposer.obs_encoder.eval()
1171
+ cache_t, cache_v = self._precompute_global_cond_cache(
1172
+ train_dataset=dataset,
1173
+ val_dataset=val_dataset,
1174
+ proposer=cache_proposer,
1175
+ n_frames=cache_n_frames,
1176
+ device=device,
1177
+ accelerator=accelerator,
1178
+ parl_cfg=cfg.parl,
1179
+ )
1180
+ self._parl_global_cond_cache = cache_t
1181
+ self._parl_global_cond_valid = cache_v
1182
+ accelerator.wait_for_everyone()
1183
+
1184
+ # save batch for sampling
1185
+ train_sampling_batch = None
1186
+
1187
+ # PA-RL relabel cache lives on disk for resumability + sanity
1188
+ # inspection ("did the optimized chunks actually move away from
1189
+ # the BC chunks?"). The in-memory swap into the train / val
1190
+ # datasets happens immediately afterwards so dataloader workers
1191
+ # spawned for the next epoch see the new data via the dataset
1192
+ # they hold a reference to.
1193
+ parl_relabel_every = int(OmegaConf.select(
1194
+ cfg, 'parl.relabel_every', default=1)) if parl_enabled else 0
1195
+ parl_save_cache = bool(OmegaConf.select(
1196
+ cfg, 'parl.save_cache_to_disk', default=True)) if parl_enabled else False
1197
+ # Pre-compute per-frame array shape so the relabel helper can
1198
+ # allocate without re-reading the replay buffer. ``dataset``
1199
+ # is the train split here; both train + val share the same
1200
+ # n_frames since they share the underlying ReplayBuffer.
1201
+ parl_n_frames = int(getattr(dataset, 'n_frames_total', 0))
1202
+ parl_action_horizon = int(getattr(dataset, 'action_horizon', 0))
1203
+ parl_action_dim = int(getattr(dataset, 'action_dim', 0))
1204
+
1205
+ if cfg.training.debug:
1206
+ cfg.training.num_epochs = 2
1207
+ cfg.training.max_train_steps = 3
1208
+ cfg.training.max_val_steps = 3
1209
+ cfg.training.rollout_every = 1
1210
+ cfg.training.checkpoint_every = 1
1211
+ cfg.training.val_every = 1
1212
+ cfg.training.sample_every = 1
1213
+
1214
+ # training loop
1215
+ log_path = os.path.join(self.output_dir, 'logs.json.txt')
1216
+ with JsonLogger(log_path) as json_logger:
1217
+ for local_epoch_idx in range(cfg.training.num_epochs):
1218
+ # ========= PA-RL: optional relabel pass ==========
1219
+ # Runs at the start of each epoch where the cycle hits
1220
+ # (epoch 0 included, so we bootstrap from the BC ckpt's
1221
+ # weights). Single-rank: only the main process samples
1222
+ # / scores; other ranks wait at the barrier and then
1223
+ # all ranks load the freshly-written cache file.
1224
+ if (
1225
+ parl_enabled
1226
+ and self.q_critic is not None
1227
+ and parl_relabel_every > 0
1228
+ and (self.epoch % parl_relabel_every) == 0
1229
+ ):
1230
+ parl_actions = None
1231
+ parl_valid = None
1232
+ parl_info = {}
1233
+ if accelerator.is_main_process:
1234
+ # Use EMA shadow as the proposer when available
1235
+ # (more stable and matches inference); fall
1236
+ # back to the unwrapped main model otherwise.
1237
+ proposer_name = str(OmegaConf.select(
1238
+ cfg, 'parl.proposer', default='ema'))
1239
+ if proposer_name == 'ema' and self.ema_model is not None:
1240
+ proposer = self.ema_model
1241
+ else:
1242
+ proposer = accelerator.unwrap_model(self.model)
1243
+ accelerator.print(
1244
+ f"[PA-RL] starting relabel for epoch={self.epoch} "
1245
+ f"(proposer={proposer_name})"
1246
+ )
1247
+ parl_actions, parl_valid, parl_info = self._parl_relabel(
1248
+ train_dataset=dataset,
1249
+ val_dataset=val_dataset,
1250
+ proposer=proposer,
1251
+ critic=self.q_critic,
1252
+ n_frames=parl_n_frames,
1253
+ action_horizon=parl_action_horizon,
1254
+ action_dim=parl_action_dim,
1255
+ device=device,
1256
+ accelerator=accelerator,
1257
+ parl_cfg=cfg.parl,
1258
+ state_embed_cache=self._parl_state_embed_cache,
1259
+ state_embed_valid=self._parl_state_embed_valid,
1260
+ global_cond_cache=self._parl_global_cond_cache,
1261
+ global_cond_valid=self._parl_global_cond_valid,
1262
+ )
1263
+ if parl_save_cache:
1264
+ cache_path = os.path.join(
1265
+ self.output_dir,
1266
+ f'relabel_cache_epoch{self.epoch:04d}.npy',
1267
+ )
1268
+ valid_path = os.path.join(
1269
+ self.output_dir,
1270
+ f'relabel_valid_epoch{self.epoch:04d}.npy',
1271
+ )
1272
+ np.save(cache_path, parl_actions)
1273
+ np.save(valid_path, parl_valid)
1274
+ accelerator.print(
1275
+ f"[PA-RL] wrote cache: {cache_path} "
1276
+ f"({parl_actions.nbytes / 1e6:.1f} MB) "
1277
+ f"+ valid mask"
1278
+ )
1279
+ accelerator.wait_for_everyone()
1280
+ # Multi-rank: ranks > 0 mmap the freshly-written
1281
+ # cache from disk; rank 0 keeps its in-memory
1282
+ # array (cheaper, no roundtrip). For the v1
1283
+ # single-GPU path the rank > 0 branch is dead
1284
+ # code -- but it costs nothing and future-proofs
1285
+ # the multi-GPU path.
1286
+ if accelerator.is_main_process:
1287
+ dataset.set_action_optimized(parl_actions, parl_valid)
1288
+ val_dataset.set_action_optimized(parl_actions, parl_valid)
1289
+ elif parl_save_cache:
1290
+ cache_path = os.path.join(
1291
+ self.output_dir,
1292
+ f'relabel_cache_epoch{self.epoch:04d}.npy',
1293
+ )
1294
+ valid_path = os.path.join(
1295
+ self.output_dir,
1296
+ f'relabel_valid_epoch{self.epoch:04d}.npy',
1297
+ )
1298
+ actions_mm = np.load(cache_path, mmap_mode='r')
1299
+ valid_mm = np.load(valid_path, mmap_mode='r').astype(bool)
1300
+ dataset.set_action_optimized(actions_mm, valid_mm)
1301
+ val_dataset.set_action_optimized(actions_mm, valid_mm)
1302
+ accelerator.wait_for_everyone()
1303
+
1304
+ # Rebuild dataloaders so any persistent_workers
1305
+ # spawn fresh and pick up the swapped cache via
1306
+ # the dataset reference. Re-prepare with
1307
+ # accelerator so DistributedSampler etc. is
1308
+ # re-applied for multi-GPU.
1309
+ fresh_train_loader = DataLoader(dataset, **cfg.dataloader)
1310
+ fresh_val_loader = DataLoader(val_dataset, **cfg.val_dataloader)
1311
+ train_dataloader, val_dataloader = accelerator.prepare(
1312
+ fresh_train_loader, fresh_val_loader,
1313
+ )
1314
+ if accelerator.is_main_process and parl_info:
1315
+ accelerator.log(
1316
+ {**parl_info, 'epoch': self.epoch},
1317
+ step=self.global_step,
1318
+ )
1319
+
1320
+ self.model.train()
1321
+
1322
+ step_log = dict()
1323
+ # ========= train for this epoch ==========
1324
+ if cfg.training.freeze_encoder:
1325
+ # ``self.model`` is a DDP wrapper after
1326
+ # accelerator.prepare(), so ``obs_encoder`` lives on
1327
+ # the inner module. Use accelerator.unwrap_model so
1328
+ # this works in both single-GPU (no wrapper) and
1329
+ # multi-GPU DDP. eval() + requires_grad_(False)
1330
+ # mutate state in place; DDP forwards the
1331
+ # frozen-grad signal through the wrapper.
1332
+ _unwrapped = accelerator.unwrap_model(self.model)
1333
+ _unwrapped.obs_encoder.eval()
1334
+ _unwrapped.obs_encoder.requires_grad_(False)
1335
+
1336
+ train_losses = list()
1337
+ with tqdm.tqdm(train_dataloader, desc=f"Training epoch {self.epoch}",
1338
+ leave=False, mininterval=cfg.training.tqdm_interval_sec) as tepoch:
1339
+ for batch_idx, batch in enumerate(tepoch):
1340
+ # device transfer
1341
+ batch = dict_apply(batch, lambda x: x.to(device, non_blocking=True))
1342
+
1343
+ # always use the latest batch
1344
+ train_sampling_batch = batch
1345
+
1346
+ # compute loss
1347
+ raw_loss = self.model(batch)
1348
+ loss = raw_loss / cfg.training.gradient_accumulate_every
1349
+ loss.backward()
1350
+
1351
+ # step optimizer
1352
+ if self.global_step % cfg.training.gradient_accumulate_every == 0:
1353
+ self.optimizer.step()
1354
+ self.optimizer.zero_grad()
1355
+ lr_scheduler.step()
1356
+
1357
+ # update ema
1358
+ if cfg.training.use_ema:
1359
+ ema.step(accelerator.unwrap_model(self.model))
1360
+
1361
+ # logging
1362
+ raw_loss_cpu = raw_loss.item()
1363
+ tepoch.set_postfix(loss=raw_loss_cpu, refresh=False)
1364
+ train_losses.append(raw_loss_cpu)
1365
+ step_log = {
1366
+ 'train_loss': raw_loss_cpu,
1367
+ 'global_step': self.global_step,
1368
+ 'epoch': self.epoch,
1369
+ 'lr': lr_scheduler.get_last_lr()[0]
1370
+ }
1371
+
1372
+ # Log bbox mask channel weight norms. Sample frequently
1373
+ # at the start so the early growth (from zero) is
1374
+ # visible on the wandb curve, then less often once
1375
+ # training is settled. Cheap (a few tensor norms) but
1376
+ # we still gate by step to avoid spamming.
1377
+ if accelerator.is_main_process:
1378
+ log_every = (10 if self.global_step < 500 else 50)
1379
+ if (self.global_step % log_every) == 0:
1380
+ _unwrapped = accelerator.unwrap_model(self.model)
1381
+ step_log.update(_bbox_mask_weight_metrics(_unwrapped))
1382
+ step_log.update(_awr_metrics(_unwrapped))
1383
+ step_log.update(_parl_metrics(_unwrapped))
1384
+
1385
+ is_last_batch = (batch_idx == (len(train_dataloader)-1))
1386
+ if not is_last_batch:
1387
+ # log of last step is combined with validation and rollout
1388
+ accelerator.log(step_log, step=self.global_step)
1389
+ json_logger.log(step_log)
1390
+ self.global_step += 1
1391
+
1392
+ if (cfg.training.max_train_steps is not None) \
1393
+ and batch_idx >= (cfg.training.max_train_steps-1):
1394
+ break
1395
+
1396
+ # at the end of each epoch
1397
+ # replace train_loss with epoch average
1398
+ train_loss = np.mean(train_losses)
1399
+ step_log['train_loss'] = train_loss
1400
+
1401
+ # End-of-epoch snapshot of the mask channel weights. Always
1402
+ # emitted (no throttling) so the wandb x-axis "epoch" has a
1403
+ # clean per-epoch curve to chart alongside train_loss.
1404
+ if accelerator.is_main_process:
1405
+ _unwrapped = accelerator.unwrap_model(self.model)
1406
+ step_log.update(_bbox_mask_weight_metrics(_unwrapped))
1407
+ step_log.update(_awr_metrics(_unwrapped))
1408
+ step_log.update(_parl_metrics(_unwrapped))
1409
+
1410
+ # ========= eval for this epoch ==========
1411
+ policy = accelerator.unwrap_model(self.model)
1412
+ if cfg.training.use_ema:
1413
+ policy = self.ema_model
1414
+ policy.eval()
1415
+
1416
+ # run rollout
1417
+ if (self.epoch % cfg.training.rollout_every) == 0:
1418
+ runner_log = env_runner.run(policy)
1419
+ # log all
1420
+ step_log.update(runner_log)
1421
+
1422
+ # run validation
1423
+ # Runs on main rank only (other ranks idle through here, then
1424
+ # rejoin at accelerator.log below). Uses the unwrapped policy
1425
+ # (EMA if enabled) so it reflects what we'd actually deploy.
1426
+ if (self.epoch % cfg.training.val_every) == 0 and len(val_dataloader) > 0 and accelerator.is_main_process:
1427
+ with torch.no_grad():
1428
+ val_losses = list()
1429
+ with tqdm.tqdm(val_dataloader, desc=f"Validation epoch {self.epoch}",
1430
+ leave=False, mininterval=cfg.training.tqdm_interval_sec) as tepoch:
1431
+ for batch_idx, batch in enumerate(tepoch):
1432
+ batch = dict_apply(batch, lambda x: x.to(device, non_blocking=True))
1433
+ # policy is already the unwrapped (EMA if
1434
+ # enabled) module — its forward() returns the
1435
+ # same diffusion loss the train step uses.
1436
+ loss = policy(batch)
1437
+ val_losses.append(loss.item())
1438
+ if (cfg.training.max_val_steps is not None) \
1439
+ and batch_idx >= (cfg.training.max_val_steps-1):
1440
+ break
1441
+ if len(val_losses) > 0:
1442
+ step_log['val_loss'] = float(np.mean(val_losses))
1443
+
1444
+ def _action_mse_components(pred, gt):
1445
+ """Compute (full, pos, rot, width) MSE components from
1446
+ (B, T, A) tensors viewed as (B, T, num_robot, 10).
1447
+ """
1448
+ B, T, _ = pred.shape
1449
+ pred_v = pred.view(B, T, -1, 10)
1450
+ gt_v = gt.view(B, T, -1, 10)
1451
+ return (
1452
+ torch.nn.functional.mse_loss(pred_v, gt_v),
1453
+ torch.nn.functional.mse_loss(pred_v[..., :3], gt_v[..., :3]),
1454
+ torch.nn.functional.mse_loss(pred_v[..., 3:9], gt_v[..., 3:9]),
1455
+ torch.nn.functional.mse_loss(pred_v[..., 9], gt_v[..., 9]),
1456
+ )
1457
+
1458
+ def log_action_mse(step_log, category, pred_action, batch):
1459
+ """Log action MSE components.
1460
+
1461
+ PA-RL changes this from the original "vs dataset action"
1462
+ behavior. The dataset for fine-tuning contains failure
1463
+ rollouts (misses, corrections) we explicitly do NOT want
1464
+ to imitate; we are training the policy to imitate the
1465
+ OPTIMIZED action targets emitted by the relabel pass.
1466
+ Therefore the primary action_mse metric is computed
1467
+ against ``batch['action_optimized']`` masked to samples
1468
+ where ``has_action_optimized`` is set. The "vs dataset"
1469
+ version is still logged under ``*_dataset`` because the
1470
+ GAP between the two is the cleanest drift signal we
1471
+ have without an expert-only val split:
1472
+ * primary low + dataset stable: good progress.
1473
+ * primary low + dataset rising: optimized targets
1474
+ drifting from BC (turn anchor up, or critic has
1475
+ blind spots).
1476
+ * primary rising: distillation failing.
1477
+
1478
+ For categories without an optimized cache yet (e.g. on
1479
+ the very first epoch before relabel runs, or with PA-RL
1480
+ disabled), the primary metric falls back to the dataset
1481
+ action so the metric remains continuous across modes.
1482
+ """
1483
+ gt_dataset = batch['action']
1484
+ full_d, pos_d, rot_d, w_d = _action_mse_components(
1485
+ pred_action, gt_dataset)
1486
+ step_log[f'{category}_action_mse_error_dataset'] = full_d
1487
+ step_log[f'{category}_action_mse_error_pos_dataset'] = pos_d
1488
+ step_log[f'{category}_action_mse_error_rot_dataset'] = rot_d
1489
+ step_log[f'{category}_action_mse_error_width_dataset'] = w_d
1490
+
1491
+ gt_opt = batch.get('action_optimized', None)
1492
+ has_opt = batch.get('has_action_optimized', None)
1493
+ use_opt = (
1494
+ gt_opt is not None
1495
+ and has_opt is not None
1496
+ and bool((has_opt > 0.5).any().item())
1497
+ )
1498
+ if use_opt:
1499
+ mask = (has_opt > 0.5).view(-1)
1500
+ pred_m = pred_action[mask]
1501
+ gt_m = gt_opt[mask]
1502
+ full_o, pos_o, rot_o, w_o = _action_mse_components(
1503
+ pred_m, gt_m)
1504
+ # Primary monitor target: how well the deployed
1505
+ # (full DDIM rollout) policy matches the actual
1506
+ # training target, on the subset that has one.
1507
+ step_log[f'{category}_action_mse_error'] = full_o
1508
+ step_log[f'{category}_action_mse_error_pos'] = pos_o
1509
+ step_log[f'{category}_action_mse_error_rot'] = rot_o
1510
+ step_log[f'{category}_action_mse_error_width'] = w_o
1511
+ else:
1512
+ # Pre-relabel or non-PARL run: optimized cache not
1513
+ # populated for this category. Use the dataset
1514
+ # comparison as the primary metric so monitor_key
1515
+ # stays well-defined and the wandb plot is
1516
+ # continuous.
1517
+ step_log[f'{category}_action_mse_error'] = full_d
1518
+ step_log[f'{category}_action_mse_error_pos'] = pos_d
1519
+ step_log[f'{category}_action_mse_error_rot'] = rot_d
1520
+ step_log[f'{category}_action_mse_error_width'] = w_d
1521
+ # run diffusion sampling on a training batch
1522
+ if (self.epoch % cfg.training.sample_every) == 0 and accelerator.is_main_process:
1523
+ with torch.no_grad():
1524
+ # sample trajectory from training set, and evaluate difference
1525
+ batch = dict_apply(train_sampling_batch, lambda x: x.to(device, non_blocking=True))
1526
+ pred_action = policy.predict_action(batch['obs'], None)['action_pred']
1527
+ log_action_mse(step_log, 'train', pred_action, batch)
1528
+
1529
+ if len(val_dataloader) > 0:
1530
+ val_sampling_batch = next(iter(val_dataloader))
1531
+ batch = dict_apply(val_sampling_batch, lambda x: x.to(device, non_blocking=True))
1532
+ pred_action = policy.predict_action(batch['obs'], None)['action_pred']
1533
+ log_action_mse(step_log, 'val', pred_action, batch)
1534
+
1535
+ del batch
1536
+ del pred_action
1537
+
1538
+ # checkpoint
1539
+ if (self.epoch % cfg.training.checkpoint_every) == 0 and accelerator.is_main_process:
1540
+ # unwrap the model to save ckpt
1541
+ model_ddp = self.model
1542
+ self.model = accelerator.unwrap_model(self.model)
1543
+
1544
+ # checkpointing
1545
+ if cfg.checkpoint.save_last_ckpt:
1546
+ self.save_checkpoint()
1547
+ if cfg.checkpoint.save_last_snapshot:
1548
+ self.save_snapshot()
1549
+
1550
+ # sanitize metric names
1551
+ metric_dict = dict()
1552
+ for key, value in step_log.items():
1553
+ new_key = key.replace('/', '_')
1554
+ metric_dict[new_key] = value
1555
+
1556
+ # We can't copy the last checkpoint here
1557
+ # since save_checkpoint uses threads.
1558
+ # therefore at this point the file might have been empty!
1559
+ topk_ckpt_path = topk_manager.get_ckpt_path(metric_dict)
1560
+
1561
+ if topk_ckpt_path is not None:
1562
+ self.save_checkpoint(path=topk_ckpt_path)
1563
+
1564
+ # recover the DDP model
1565
+ self.model = model_ddp
1566
+ # ========= eval end for this epoch ==========
1567
+ # end of epoch
1568
+ # log of last step is combined with validation and rollout
1569
+ accelerator.log(step_log, step=self.global_step)
1570
+ json_logger.log(step_log)
1571
+ self.global_step += 1
1572
+ self.epoch += 1
1573
+
1574
+ accelerator.end_training()
1575
+
1576
+ @hydra.main(
1577
+ version_base=None,
1578
+ config_path=str(pathlib.Path(__file__).parent.parent.joinpath("config")),
1579
+ config_name=pathlib.Path(__file__).stem)
1580
+ def main(cfg):
1581
+ workspace = TrainDiffusionUnetImageWorkspace(cfg)
1582
+ workspace.run()
1583
+
1584
+ if __name__ == "__main__":
1585
+ main()