solarian-diffusion 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. diffusion_policy/__init__.py +0 -0
  2. diffusion_policy/codecs/__init__.py +0 -0
  3. diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
  4. diffusion_policy/common/__init__.py +0 -0
  5. diffusion_policy/common/checkpoint_util.py +64 -0
  6. diffusion_policy/common/json_logger.py +117 -0
  7. diffusion_policy/common/normalize_util.py +251 -0
  8. diffusion_policy/common/pose_repr_util.py +112 -0
  9. diffusion_policy/common/pytorch_util.py +82 -0
  10. diffusion_policy/common/q_critic_util.py +135 -0
  11. diffusion_policy/common/replay_buffer.py +672 -0
  12. diffusion_policy/common/sampler.py +204 -0
  13. diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
  14. diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
  15. diffusion_policy/config/legacy/task/can_image.yaml +64 -0
  16. diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
  17. diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
  18. diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
  19. diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
  20. diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
  21. diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
  22. diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
  23. diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
  24. diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
  25. diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
  26. diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
  27. diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
  28. diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
  29. diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
  30. diffusion_policy/config/legacy/task/square_image.yaml +64 -0
  31. diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
  32. diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
  33. diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
  34. diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
  35. diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
  36. diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
  37. diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
  38. diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
  39. diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
  40. diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
  41. diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
  42. diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
  43. diffusion_policy/config/legacy/task/umi.yaml +45 -0
  44. diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
  45. diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
  46. diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
  47. diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
  48. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
  49. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
  50. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
  51. diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
  52. diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
  53. diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
  54. diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
  55. diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
  56. diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
  57. diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
  58. diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
  59. diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
  60. diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
  61. diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
  62. diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
  63. diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
  64. diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
  65. diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
  66. diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
  67. diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
  68. diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
  69. diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
  70. diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
  71. diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
  72. diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
  73. diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
  74. diffusion_policy/config/rl_annotate.yaml +265 -0
  75. diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
  76. diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
  77. diffusion_policy/config/rl_annotate_v2.yaml +141 -0
  78. diffusion_policy/config/rl_annotate_v3.yaml +165 -0
  79. diffusion_policy/config/task/lift.yaml +89 -0
  80. diffusion_policy/config/task/square.yaml +92 -0
  81. diffusion_policy/config/task/tool_hang.yaml +93 -0
  82. diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
  83. diffusion_policy/config/task/umi.yaml +105 -0
  84. diffusion_policy/config/task/umi_bimanual.yaml +135 -0
  85. diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
  86. diffusion_policy/config/task/umi_image.yaml +36 -0
  87. diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
  88. diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
  89. diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
  90. diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
  91. diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
  92. diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
  93. diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
  94. diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
  95. diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
  96. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
  97. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
  98. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
  99. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
  100. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
  101. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
  102. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
  103. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
  104. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
  105. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
  106. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
  107. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
  108. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
  109. diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
  110. diffusion_policy/config/train_iql_umi.yaml +186 -0
  111. diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
  112. diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
  113. diffusion_policy/dataset/__init__.py +0 -0
  114. diffusion_policy/dataset/base_dataset.py +76 -0
  115. diffusion_policy/dataset/umi_dataset.py +652 -0
  116. diffusion_policy/env_runner/__init__.py +0 -0
  117. diffusion_policy/env_runner/base_image_runner.py +9 -0
  118. diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
  119. diffusion_policy/model/__init__.py +0 -0
  120. diffusion_policy/model/common/__init__.py +0 -0
  121. diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
  122. diffusion_policy/model/common/lr_scheduler.py +46 -0
  123. diffusion_policy/model/common/module_attr_mixin.py +14 -0
  124. diffusion_policy/model/common/normalizer.py +353 -0
  125. diffusion_policy/model/common/tensor_util.py +960 -0
  126. diffusion_policy/model/diffusion/__init__.py +0 -0
  127. diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
  128. diffusion_policy/model/diffusion/conv1d_components.py +46 -0
  129. diffusion_policy/model/diffusion/ema_model.py +88 -0
  130. diffusion_policy/model/diffusion/mask_generator.py +221 -0
  131. diffusion_policy/model/diffusion/positional_embedding.py +17 -0
  132. diffusion_policy/model/vision/__init__.py +0 -0
  133. diffusion_policy/model/vision/crop_randomizer.py +288 -0
  134. diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
  135. diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
  136. diffusion_policy/policy/__init__.py +0 -0
  137. diffusion_policy/policy/base_image_policy.py +27 -0
  138. diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
  139. diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
  140. diffusion_policy/policy/parl_action_optimizer.py +675 -0
  141. diffusion_policy/workspace/__init__.py +0 -0
  142. diffusion_policy/workspace/base_workspace.py +146 -0
  143. diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
  144. solarian_diffusion/__init__.py +32 -0
  145. solarian_diffusion/config.py +31 -0
  146. solarian_diffusion/policy_server.py +316 -0
  147. solarian_diffusion/rtc.py +272 -0
  148. solarian_diffusion/service.py +16 -0
  149. solarian_diffusion/trainer.py +213 -0
  150. solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
  151. solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
  152. solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
  153. solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
  154. solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
  155. solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
  156. solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
  157. umi/__init__.py +0 -0
  158. umi/common/__init__.py +0 -0
  159. umi/common/pose_util.py +115 -0
@@ -0,0 +1,399 @@
1
+ # =============================================================================
2
+ # PA-RL fine-tune of the BC pick_lego_pile_bbox diffusion policy.
3
+ #
4
+ # Implements offline Policy-Agnostic RL (Sobol Mark et al., 2024,
5
+ # "Policy Agnostic RL", arXiv:2412.06685) on top of an existing BC
6
+ # diffusion policy and a frozen offline IQL critic. The recipe is
7
+ # the "hybrid relabel" variant agreed in the plan:
8
+ #
9
+ # * Bootstrap relabel from the BC ckpt's EMA at epoch 0.
10
+ # * Re-relabel every `parl.relabel_every` epochs from the current
11
+ # EMA snapshot.
12
+ # * Each relabel pass: sample K chunks per state from the EMA
13
+ # proposer, rerank by min(Q1, Q2), keep top-m, run T steps of
14
+ # gradient ascent on Q with the revert-on-no-improve rule
15
+ # (Algorithm 1), then pick (argmax-Q by default).
16
+ # * Train the diffusion model with the standard DDPM eps loss
17
+ # against those optimized actions (Eq. 4.5 in the paper). NO
18
+ # reparameterized policy gradient through the denoising chain.
19
+ #
20
+ # AWR knobs are mutually exclusive with PA-RL: when the dataloader
21
+ # emits action_optimized, the policy.compute_loss path forces
22
+ # awr_active=False even if awr_beta > 0. We still ship awr_beta=0
23
+ # explicitly so the AWR branch is a no-op even on a stale ckpt that
24
+ # learned non-zero beta. The eps-anchor toward the BC ckpt
25
+ # (`awr_anchor_lambda`) is OPTIONAL and disabled by default; turn it
26
+ # on if rollouts drift twitchy as you've seen with prior AWR runs.
27
+ #
28
+ # Required CLI overrides:
29
+ #
30
+ # training.init_from_ckpt=<BC ckpt>
31
+ # training.q_critic_ckpt=<IQL ckpt>
32
+ #
33
+ # Smoke-test recipes (all override on top of this file):
34
+ # * Filtered/iterated BC sanity baseline (degenerate PA-RL):
35
+ # parl.k_samples=1 parl.T_grad_steps=0 parl.include_dataset_action=true
36
+ # (single candidate per state == dataset action; no Q optimization;
37
+ # trains the policy to imitate exactly the dataset actions every
38
+ # epoch. Should reproduce plain BC up to the EMA divergence.)
39
+ # * PA-RL global-only (matches the IDQL-style ablation in Table 4):
40
+ # parl.T_grad_steps=0
41
+ # * PA-RL local-only:
42
+ # parl.k_samples=1 parl.include_dataset_action=false parl.T_grad_steps=10
43
+ # * PA-RL with BC anchor safety net (defensive against drift):
44
+ # policy.awr_anchor_lambda=0.3
45
+ #
46
+ # Cost note: each relabel pass at K=32, m=10, T=10, DDIM=5 over
47
+ # ~100k frames finishes in a few minutes on a single A6000-class GPU.
48
+ # Relabel cache size: ~64 MB at H=16, A=10, fp32, N=100k frames.
49
+ # =============================================================================
50
+
51
+ defaults:
52
+ - task: umi_pick_single_lego_bbox
53
+ - _self_
54
+
55
+ name: train_diffusion_unet_timm_parl
56
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
57
+
58
+ shape_meta: ${task.shape_meta}
59
+ exp_name: "parl"
60
+ batch_size: 128
61
+ visual_encoder: "vit_large_patch14_clip_224"
62
+ visual_encoder_tag: "vitl"
63
+ model_type: "unet"
64
+ num_gpus: 1
65
+
66
+ mirror: ${parse_dataset_field:${task.dataset_path},mirror,no_mirror}
67
+ real_data_size: ${parse_dataset_field:${task.dataset_path},real_data_size,null}
68
+ sim_data_size: ${parse_dataset_field:${task.dataset_path},sim_data_size,null}
69
+ bbox: ${parse_dataset_field:${task.dataset_path},bbox,no_bbox}
70
+ bbox_mode: ${task.shape_meta.obs.bbox.bbox_mode}
71
+ task_name: ${parse_dataset_field:${task.dataset_path},data_task,pick_lego}
72
+
73
+ n_action_steps: 8
74
+
75
+ policy:
76
+ _target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
77
+
78
+ shape_meta: ${shape_meta}
79
+
80
+ noise_scheduler:
81
+ _target_: diffusers.DDIMScheduler
82
+ num_train_timesteps: 50
83
+ beta_start: 0.0001
84
+ beta_end: 0.02
85
+ beta_schedule: squaredcos_cap_v2
86
+ clip_sample: True
87
+ set_alpha_to_one: True
88
+ steps_offset: 0
89
+ prediction_type: epsilon
90
+
91
+ obs_encoder:
92
+ _target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
93
+ shape_meta: ${shape_meta}
94
+ model_name: ${visual_encoder}
95
+ pretrained: True
96
+ # Match the AWR config: encoder is trainable in principle but
97
+ # frozen at runtime via `training.freeze_encoder=true` so the
98
+ # IQL critic's state distribution stays the one it was calibrated
99
+ # against -- otherwise `Q(s, a)` becomes noisy as the encoder
100
+ # drifts and the relabel pass starts producing garbage targets.
101
+ frozen: False
102
+
103
+ global_pool: ''
104
+ feature_aggregation: 'attention_pool_2d'
105
+ position_encording: 'sinusoidal'
106
+ downsample_ratio: 32
107
+
108
+ # Lighter augmentations than BC for the same reason as AWR: PA-RL
109
+ # is a refinement of an existing policy, and aggressive crops
110
+ # would feed encoder-distorted obs into the relabel critic.
111
+ transforms:
112
+ - type: RandomCrop
113
+ ratio: 0.95
114
+ - _target_: torchvision.transforms.ColorJitter
115
+ brightness: 0.3
116
+ contrast: 0.4
117
+ saturation: 0.5
118
+ hue: 0.08
119
+
120
+ use_group_norm: True
121
+ share_rgb_model: False
122
+ imagenet_norm: True
123
+
124
+ num_inference_steps: 16
125
+ obs_as_global_cond: True
126
+ diffusion_step_embed_dim: 128
127
+ down_dims: [256,512,1024]
128
+ kernel_size: 5
129
+ n_groups: 8
130
+ cond_predict_scale: True
131
+ input_pertub: 0.1
132
+ train_diffusion_n_samples: 1
133
+
134
+ # ============== AWR knobs: all OFF for PA-RL ==============
135
+ # The PA-RL path replaces AWR entirely: when the batch carries an
136
+ # ``action_optimized`` chunk, ``compute_loss`` swaps the DDPM target
137
+ # to it and force-disables AWR weighting (advantage / has_advantage
138
+ # are forced to None). These zeros are belt-and-suspenders so the
139
+ # AWR branch can't fire even if a future change inadvertently leaks
140
+ # the awr keys.
141
+ awr_beta: 0.0
142
+ awr_w_max: 1.0
143
+ awr_w_min: 0.0
144
+ awr_lambda: 0.0
145
+ awr_normalize_weights: false
146
+ awr_bc_lambda: 0.0
147
+ # Optional eps-space anchor toward the frozen BC ckpt -- disabled
148
+ # by default since PA-RL's distillation target is itself derived
149
+ # from the BC policy via Q-optimization (so it stays close to the
150
+ # BC manifold by construction). Set to 0.3 if rollouts drift.
151
+ awr_anchor_lambda: 0.5
152
+
153
+ # ============== PA-RL block (consumed by the workspace) ==============
154
+ parl:
155
+ enabled: true
156
+
157
+ # Epochs between relabel passes. 1 = relabel every epoch (closest
158
+ # to the per-batch on-policy ideal in the paper). Bump to 2-3 if
159
+ # the relabel wall-clock dominates training time.
160
+ relabel_every: 1
161
+
162
+ # Action-optimization knobs from the paper (Sec 4.1, B.1):
163
+ # k=32 candidates per state, top m=10, T=10 grad steps, alpha=3e-4.
164
+ # The paper notes "larger action spaces (such as bimanual
165
+ # manipulation) might require larger gradient step sizes"; UMI's
166
+ # 10-D-per-step is medium-sized so the default alpha is fine.
167
+ k_samples: 32
168
+ m_top: 10
169
+ # T_grad_steps: 10
170
+ T_grad_steps: 2
171
+ alpha_grad: 3.0e-4
172
+
173
+ # Pick: argmax_Q over the optimized set is the recommended choice
174
+ # for narrow / multimodal datasets per Sec B.1 ("If the variance
175
+ # is too small, we find that training only with the argmax performs
176
+ # better."). Switch to "softmax" to sample from the categorical
177
+ # over Q values for diversity.
178
+ distill_mode: argmax
179
+ softmax_temperature: 1.0
180
+
181
+ # Include the DATASET action as an extra candidate before pick
182
+ # (Sec 4.3 "implementation details"). Cheap; helps when the
183
+ # dataset action is already near-optimal for that state.
184
+ include_dataset_action: true
185
+
186
+ # EMA snapshot is the "current policy" for proposing candidates.
187
+ # More stable than the main model (which is mid-optimizer-step) and
188
+ # is what the rollout / val passes use, so it matches deployment.
189
+ proposer: ema
190
+
191
+ # Cheaper DDIM steps for the relabel pass than at deployment.
192
+ # Picking 5 here (vs 16 at policy.num_inference_steps) cuts the
193
+ # diffusion sampler's wall-clock by ~3x with no measurable hit to
194
+ # candidate quality at K=32 (the rerank picks the best regardless).
195
+ proposer_inference_steps: 5
196
+
197
+ # Per-batch states to process during the relabel pass. With K=32
198
+ # we expand to (B*K, H, A) for a single batched U-Net forward, so
199
+ # B=128 means 4096 trajectories per forward -- comfortable on
200
+ # A100-80GB after the autocast(bf16) + dedup-encoder + state-embed
201
+ # cache wins. The relabel pass has fixed per-iter overhead (two
202
+ # GPU<->CPU sync points and the diffusion sampler's Python step
203
+ # loop) that does NOT scale with B, so larger B amortizes it. On
204
+ # A100-80GB the smoke run idles at ~8% memory at B=64; raising B
205
+ # is the cheapest remaining speedup.
206
+ # Lower (e.g. 32-64) on 24-40GB cards or if you OOM.
207
+ relabel_batch_size: 128
208
+ relabel_num_workers: 12
209
+
210
+ # When True (default), persist the relabel cache to
211
+ # `${output_dir}/relabel_cache_epoch{N:04d}.npy` plus a sibling
212
+ # `_valid` mask. Useful for sanity-checking ("did optimized chunks
213
+ # actually move?") and for reading from rank > 0 in a future
214
+ # multi-GPU relabel.
215
+ save_cache_to_disk: true
216
+
217
+ # v2 forward-compat: if False, also train the IQL critic with
218
+ # optimized-action TD targets per Algorithm 2. Not implemented
219
+ # in v1 -- leaving this knob in place documents the intent.
220
+ freeze_critic: true
221
+
222
+ # When True (and freeze_critic=true), precompute critic.state_embed
223
+ # for every dataset frame ONCE after the critic is loaded and reuse
224
+ # it across all relabel cycles. Eliminates the per-batch IQL ViT-L
225
+ # forward inside optimize_actions; replaced by an O(B) index_select
226
+ # from a pinned CPU tensor. Builds in ~3-5 min on A100 for ~170k
227
+ # frames; saves 25-30% of the per-relabel wall-clock thereafter.
228
+ # Optional separate batch size / worker count for the build pass
229
+ # (defaults to the relabel knobs); the build is single-rank and
230
+ # encoder-bound so larger batches help when GPU memory permits.
231
+ cache_state_embed: true
232
+
233
+ # When True (and training.freeze_encoder=true), precompute the
234
+ # PROPOSER's obs_encoder output for every dataset frame ONCE and
235
+ # reuse it across all relabel cycles. Eliminates the second big
236
+ # ViT-L forward inside ``propose_actions`` (the proposer's encoder
237
+ # is the same architecture as the critic's). With both caches on,
238
+ # the per-iter relabel cost has zero ViT-L forwards left -- only
239
+ # the U-Net's K diffusion samples and tiny MLP heads. Adds a
240
+ # one-time ~3-5 min build pass with the same precompute_batch_size
241
+ # as the state_embed cache.
242
+ # Auto-disabled when training.freeze_encoder=false (the cache
243
+ # would be invalidated as soon as the encoder updates).
244
+ cache_global_cond: true
245
+
246
+ # The build pass is encoder-only (no U-Net, no local-opt), so it
247
+ # tolerates a much bigger batch than the relabel pass on the same
248
+ # card. 256 fits comfortably on A100-80GB; halve on 24-40GB cards.
249
+ precompute_batch_size: 256
250
+ precompute_num_workers: 12
251
+
252
+ ema:
253
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
254
+ update_after_step: 0
255
+ inv_gamma: 1.0
256
+ power: 0.75
257
+ min_value: 0.0
258
+ max_value: 0.9999
259
+
260
+ dataloader:
261
+ batch_size: ${batch_size}
262
+ num_workers: 8
263
+ shuffle: True
264
+ pin_memory: True
265
+ # PA-RL rebuilds the dataloader after each relabel pass so workers
266
+ # spawn fresh and pick up the swapped optimized-action cache. With
267
+ # persistent_workers=True the OLD workers would still hold a
268
+ # forked copy of the dataset with stale (or absent) cache, which is
269
+ # exactly what we don't want.
270
+ persistent_workers: False
271
+ prefetch_factor: 4
272
+
273
+ val_dataloader:
274
+ batch_size: ${batch_size}
275
+ num_workers: 8
276
+ shuffle: False
277
+ pin_memory: True
278
+ persistent_workers: False
279
+ prefetch_factor: 4
280
+
281
+ optimizer:
282
+ _target_: torch.optim.AdamW
283
+ # 5e-5: paper's distillation LR for kitchen-style narrow data
284
+ # (Appendix B.1). Up from AWR's 1e-5 because PA-RL's relabel
285
+ # targets are themselves moving toward the optimum -- the policy
286
+ # needs to keep up. Drop to 1e-5 if you observe oscillating
287
+ # val_loss between relabel cycles.
288
+ lr: 5.0e-5
289
+ betas: [0.95, 0.999]
290
+ eps: 1.0e-8
291
+ weight_decay: 1.0e-6
292
+
293
+ training:
294
+ device: "cuda:0"
295
+ seed: 42
296
+ debug: False
297
+ resume: False
298
+ lr_scheduler: cosine
299
+ lr_warmup_steps: 100
300
+ # PA-RL needs more epochs than AWR: each relabel pass only nudges
301
+ # the policy, and you need several cycles for the policy ->
302
+ # relabel -> policy loop to converge. Bump to 50 if the val_loss
303
+ # is still trending down at epoch 30.
304
+ num_epochs: 30
305
+ gradient_accumulate_every: 1
306
+ use_ema: True
307
+ # Same rationale as AWR: the IQL critic was trained against the
308
+ # BC encoder's features, so we freeze the encoder during PA-RL
309
+ # so Q(s, a) values stay calibrated.
310
+ freeze_encoder: True
311
+
312
+ # BC checkpoint to initialize from. SET BEFORE RUNNING.
313
+ init_from_ckpt: null
314
+
315
+ # NEW (PA-RL): IQL critic checkpoint. Same path semantics as
316
+ # init_from_ckpt: optimizer + step counters are NOT loaded; only
317
+ # the critic's state_dict goes onto the module. SET BEFORE
318
+ # RUNNING (override on CLI:
319
+ # training.q_critic_ckpt=/path/to/iql.ckpt
320
+ # ).
321
+ q_critic_ckpt: null
322
+
323
+ rollout_every: 10
324
+ checkpoint_every: 1
325
+ val_every: 1
326
+ sample_every: 1
327
+ max_train_steps: null
328
+ max_val_steps: null
329
+ tqdm_interval_sec: 1.0
330
+
331
+ logging:
332
+ project: umi
333
+ resume: False
334
+ mode: online
335
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
336
+ tags:
337
+ - "${name}"
338
+ - "${task_name}"
339
+ - "${exp_name}"
340
+ - "parl"
341
+ - "${visual_encoder_tag}"
342
+ - "batch_size:${eval:'${batch_size} * ${num_gpus}'}"
343
+ - "num_gpus:${num_gpus}"
344
+ - "${model_type}"
345
+ - "${eval:'\"${task.dataset_path}\".split(\"/\")[-1].rsplit(\".\",1)[0]'}"
346
+ - "${mirror}"
347
+ - "${eval:'\"real_size:%d\" % ${real_data_size} if ${real_data_size} is not None else \"\"'}"
348
+ - "${eval:'\"sim_size:%d\" % ${sim_data_size} if ${sim_data_size} is not None else \"\"'}"
349
+ - "${eval:'(\"real\" if ${real_data_size} > 0 else \"no_real\") if ${real_data_size} is not None else \"\"'}"
350
+ - "${eval:'(\"sim\" if ${sim_data_size} > 0 else \"no_sim\") if ${sim_data_size} is not None else \"\"'}"
351
+ - "${bbox}"
352
+ - "bbox_mode:${bbox_mode}"
353
+ - "parl_K:${parl.k_samples}"
354
+ - "parl_m:${parl.m_top}"
355
+ - "parl_T:${parl.T_grad_steps}"
356
+ - "parl_alpha:${parl.alpha_grad}"
357
+ - "parl_pick:${parl.distill_mode}"
358
+ - "parl_proposer:${parl.proposer}"
359
+ - "parl_relabel_every:${parl.relabel_every}"
360
+ - "parl_include_data_action:${parl.include_dataset_action}"
361
+ - "parl_anchor_lambda:${policy.awr_anchor_lambda}"
362
+ id: null
363
+ group: ${task_name}_${model_type}_parl
364
+
365
+ checkpoint:
366
+ topk:
367
+ # Track val_loss: with PA-RL active, val_loss is the unweighted
368
+ # DDPM loss against the OPTIMIZED action target (matches train
369
+ # objective). Lower = better distillation of the Q-optimal chunk.
370
+ #
371
+ # NOTE: val_loss alone is unsafe when policy.awr_anchor_lambda
372
+ # is 0 -- the loss can decrease while the optimized targets
373
+ # drift away from the demonstration manifold in critic-blind
374
+ # dims (rotation, gripper width, Z when reward is sparse-
375
+ # success). The recommended setup pairs this monitor with
376
+ # policy.awr_anchor_lambda >= 0.3, which architecturally bounds
377
+ # drift; under that constraint, lower val_loss == better policy.
378
+ # If you ever turn the anchor off, switch monitor_key to a
379
+ # drift-aware metric (see the parl/val_action_pos_expert hook
380
+ # below, gated by training.parl.expert_episode_filter).
381
+ monitor_key: val_loss
382
+ mode: min
383
+ k: 10
384
+ format_str: '{task_name}-parl-{model_type}-{visual_encoder}-batch_{total_batch}-{dataset_file_name}-epoch_{epoch:04d}-val_loss_{val_loss:.4f}.ckpt'
385
+ save_last_ckpt: True
386
+ save_last_snapshot: False
387
+
388
+ multi_run:
389
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
390
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
391
+
392
+ hydra:
393
+ job:
394
+ override_dirname: ${name}
395
+ run:
396
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
397
+ sweep:
398
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
399
+ subdir: ${hydra.job.num}
@@ -0,0 +1,258 @@
1
+ defaults:
2
+ - _self_
3
+ - task: umi_pick_lego
4
+
5
+ name: train_diffusion_unet_timm
6
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
7
+
8
+ # task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+ batch_size: 128
12
+ # visual_encoder: "vit_base_patch16_clip_224.openai"
13
+ # visual_encoder_tag: "vitb"
14
+ visual_encoder: "resnet34.a1_in1k"
15
+ visual_encoder_tag: "resnet34"
16
+ model_type: "unet"
17
+ # this will be overridden at call time
18
+ num_gpus: 1
19
+
20
+
21
+ # Parsed from the dataset file name's hyphen-separated tokens.
22
+ # eg: pick_lego-black_mirror-real_200-sim_200-no_bbox.zarr.zip
23
+ # Resolver: train.py::parse_dataset_field. Last arg is the fallback default.
24
+ # mirror options: black_mirror, mirror, swap_mirror, no_mirror
25
+ mirror: ${parse_dataset_field:${task.dataset_path},mirror,no_mirror}
26
+ # real_data_size / sim_data_size are null when the filename has no real_<N>/sim_<N>
27
+ # token; the wandb tags below skip themselves in that case.
28
+ real_data_size: ${parse_dataset_field:${task.dataset_path},real_data_size,null}
29
+ sim_data_size: ${parse_dataset_field:${task.dataset_path},sim_data_size,null}
30
+ bbox: ${parse_dataset_field:${task.dataset_path},bbox,no_bbox}
31
+ task_name: ${parse_dataset_field:${task.dataset_path},data_task,pick_lego}
32
+
33
+ # Override dataset path for fine-tuning
34
+ # This will override the dataset_path in task/umi.yaml
35
+ # Uncomment and set to your new dataset path:
36
+ # task.dataset_path: "path/to/your/new/dataset.zarr.zip"
37
+
38
+ n_action_steps: 8
39
+
40
+ policy:
41
+ _target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
42
+
43
+ shape_meta: ${shape_meta}
44
+
45
+ noise_scheduler:
46
+ _target_: diffusers.DDIMScheduler
47
+ num_train_timesteps: 50
48
+ beta_start: 0.0001
49
+ beta_end: 0.02
50
+ # beta_schedule is important
51
+ # this is the best we found
52
+ beta_schedule: squaredcos_cap_v2
53
+ clip_sample: True
54
+ set_alpha_to_one: True
55
+ steps_offset: 0
56
+ prediction_type: epsilon # or sample
57
+
58
+ obs_encoder:
59
+ _target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
60
+ shape_meta: ${shape_meta}
61
+
62
+ ##### from scratch #####
63
+ # model_name: 'vit_base_patch16_224'
64
+ # model_name: 'resnet34'
65
+ # model_name: 'vit_tiny_patch16_224'
66
+ # model_name: 'efficientnet_b0'
67
+ # model_name: 'efficientnet_b3'
68
+ # pretrained: False
69
+ # frozen: False
70
+
71
+ ##### from scratch #####
72
+ # model_name: 'resnet34.a1_in1k'
73
+ model_name: ${visual_encoder}
74
+ # model_name: 'vit_large_patch14_clip_224' # Changed to match checkpoint cup_wild_vit_l_1img.ckpt (ViT-L with CLIP weights)
75
+ # model_name: 'convnext_base.clip_laion2b_augreg_ft_in12k'
76
+ pretrained: True
77
+ frozen: False
78
+
79
+
80
+ global_pool: ''
81
+
82
+ # 'avg' or 'max' or 'soft_attention' or 'spatial_embedding' or 'transformer' or 'attention_pool_2d'
83
+ feature_aggregation: 'attention_pool_2d'
84
+ # feature_aggregation: null
85
+ position_encording: 'sinusoidal' # 'learnable' or 'sinusoidal'. it only works for transformer
86
+
87
+ # it only works for resnet. 32 (7x7) or 16 (14x14)
88
+ downsample_ratio: 32
89
+
90
+ transforms:
91
+ - type: RandomCrop
92
+ ratio: 0.95
93
+ # RandomRotation is not compatible with BF16
94
+ # - _target_: torchvision.transforms.RandomRotation
95
+ # degrees:
96
+ # - -5.0
97
+ # - 5.0
98
+ # expand: false
99
+ - _target_: torchvision.transforms.ColorJitter
100
+ brightness: 0.3
101
+ contrast: 0.4
102
+ saturation: 0.5
103
+ hue: 0.08
104
+ # - _target_: torchvision.transforms.RandomPerspective
105
+ # distortion_scale: 0.5
106
+ # p: 1.0
107
+ # - _target_: torchvision.transforms.ElasticTransform
108
+ # alpha: 50.0
109
+ # sigma: 5.0
110
+ # - _target_: torchvision.transforms.RandomPerspective
111
+ # distortion_scale: 0.5
112
+ # p: 1.0
113
+ # - _target_: torchvision.transforms.ElasticTransform
114
+ # alpha: 50.0
115
+ # sigma: 5.0
116
+ # - _target_: torchvision.transforms.RandomCrop
117
+ # size: 192
118
+
119
+
120
+ use_group_norm: True
121
+ share_rgb_model: False
122
+ imagenet_norm: True
123
+
124
+ num_inference_steps: 16
125
+ obs_as_global_cond: True
126
+ diffusion_step_embed_dim: 128
127
+ down_dims: [256,512,1024]
128
+ kernel_size: 5
129
+ n_groups: 8
130
+ cond_predict_scale: True
131
+ input_pertub: 0.1 # reference: https://github.com/forever208/DDPM-IP
132
+ # train multiple diffusion steps per obs encoder forward pass
133
+ # doing this too much will cause instability!
134
+ train_diffusion_n_samples: 1
135
+
136
+ ema:
137
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
138
+ update_after_step: 0
139
+ inv_gamma: 1.0
140
+ power: 0.75
141
+ min_value: 0.0
142
+ max_value: 0.9999
143
+
144
+ dataloader:
145
+ batch_size: ${batch_size}
146
+ num_workers: 8
147
+ shuffle: True
148
+ pin_memory: True
149
+ persistent_workers: True
150
+ prefetch_factor: 4
151
+
152
+ val_dataloader:
153
+ batch_size: ${batch_size}
154
+ num_workers: 8
155
+ shuffle: False
156
+ pin_memory: True
157
+ persistent_workers: True
158
+ prefetch_factor: 4
159
+
160
+ # dataloader:
161
+ # batch_size: 64
162
+ # num_workers: 16
163
+ # shuffle: True
164
+ # pin_memory: True
165
+ # persistent_workers: True
166
+
167
+ # val_dataloader:
168
+ # batch_size: 64
169
+ # num_workers: 16
170
+ # shuffle: False
171
+ # pin_memory: True
172
+ # persistent_workers: True
173
+
174
+ optimizer:
175
+ _target_: torch.optim.AdamW
176
+ lr: 3.0e-4
177
+ betas: [0.95, 0.999]
178
+ eps: 1.0e-8
179
+ weight_decay: 1.0e-6
180
+
181
+ training:
182
+ device: "cuda:0"
183
+ seed: 42
184
+ debug: False
185
+ resume: False
186
+ # optimization
187
+ lr_scheduler: cosine
188
+ lr_warmup_steps: 500
189
+ num_epochs: 600
190
+ gradient_accumulate_every: 1
191
+ # EMA destroys performance when used with BatchNorm
192
+ # replace BatchNorm with GroupNorm.
193
+ use_ema: True
194
+ freeze_encoder: False
195
+ # training loop control
196
+ # in epochs
197
+ rollout_every: 10
198
+ checkpoint_every: 10
199
+ val_every: 1
200
+ sample_every: 5
201
+ # steps per epoch
202
+ max_train_steps: null
203
+ max_val_steps: null
204
+ # misc
205
+ tqdm_interval_sec: 1.0
206
+
207
+ logging:
208
+ project: umi
209
+ resume: False
210
+ mode: online
211
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
212
+ tags:
213
+ - "${name}"
214
+ - "${task_name}"
215
+ - "${exp_name}"
216
+ # visual encoder could be vit_base_patch16_clip_224.openai, resnet34, etc.
217
+ - "${visual_encoder_tag}"
218
+ # batch size for each gpu
219
+ - "batch_size:${eval:'${batch_size} * ${num_gpus}'}"
220
+ - "num_gpus:${num_gpus}"
221
+ # model type could be unet or transformer
222
+ - "${model_type}"
223
+ # dataset file name (joined; substring-searchable on wandb)
224
+ # Note: full ${task.dataset_path} dropped — wandb tags max 64 chars.
225
+ - "${eval:'\"${task.dataset_path}\".split(\"/\")[-1].rsplit(\".\",1)[0]'}"
226
+ - "${mirror}"
227
+ # Only emit real_/sim_ tags when the dataset filename actually carries
228
+ # a real_<N> / sim_<N> token (otherwise the resolver returns None and we
229
+ # render an empty string here, which is filtered out before wandb init).
230
+ - "${eval:'\"real_size:%d\" % ${real_data_size} if ${real_data_size} is not None else \"\"'}"
231
+ - "${eval:'\"sim_size:%d\" % ${sim_data_size} if ${sim_data_size} is not None else \"\"'}"
232
+ - "${eval:'(\"real\" if ${real_data_size} > 0 else \"no_real\") if ${real_data_size} is not None else \"\"'}"
233
+ - "${eval:'(\"sim\" if ${sim_data_size} > 0 else \"no_sim\") if ${sim_data_size} is not None else \"\"'}"
234
+ - "${bbox}"
235
+ id: null
236
+ group: ${task_name}_${model_type}
237
+
238
+ checkpoint:
239
+ topk:
240
+ monitor_key: train_loss
241
+ mode: min
242
+ k: 20
243
+ format_str: '{task_name}-{model_type}-{visual_encoder}-batch_{total_batch}-{dataset_file_name}-epoch_{epoch:04d}-train_loss_{train_loss:.3f}.ckpt'
244
+ save_last_ckpt: True
245
+ save_last_snapshot: False
246
+
247
+ multi_run:
248
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
249
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
250
+
251
+ hydra:
252
+ job:
253
+ override_dirname: ${name}
254
+ run:
255
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
256
+ sweep:
257
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
258
+ subdir: ${hydra.job.num}