solarian-diffusion 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. diffusion_policy/__init__.py +0 -0
  2. diffusion_policy/codecs/__init__.py +0 -0
  3. diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
  4. diffusion_policy/common/__init__.py +0 -0
  5. diffusion_policy/common/checkpoint_util.py +64 -0
  6. diffusion_policy/common/json_logger.py +117 -0
  7. diffusion_policy/common/normalize_util.py +251 -0
  8. diffusion_policy/common/pose_repr_util.py +112 -0
  9. diffusion_policy/common/pytorch_util.py +82 -0
  10. diffusion_policy/common/q_critic_util.py +135 -0
  11. diffusion_policy/common/replay_buffer.py +672 -0
  12. diffusion_policy/common/sampler.py +204 -0
  13. diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
  14. diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
  15. diffusion_policy/config/legacy/task/can_image.yaml +64 -0
  16. diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
  17. diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
  18. diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
  19. diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
  20. diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
  21. diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
  22. diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
  23. diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
  24. diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
  25. diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
  26. diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
  27. diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
  28. diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
  29. diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
  30. diffusion_policy/config/legacy/task/square_image.yaml +64 -0
  31. diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
  32. diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
  33. diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
  34. diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
  35. diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
  36. diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
  37. diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
  38. diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
  39. diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
  40. diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
  41. diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
  42. diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
  43. diffusion_policy/config/legacy/task/umi.yaml +45 -0
  44. diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
  45. diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
  46. diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
  47. diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
  48. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
  49. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
  50. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
  51. diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
  52. diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
  53. diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
  54. diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
  55. diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
  56. diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
  57. diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
  58. diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
  59. diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
  60. diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
  61. diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
  62. diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
  63. diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
  64. diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
  65. diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
  66. diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
  67. diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
  68. diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
  69. diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
  70. diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
  71. diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
  72. diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
  73. diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
  74. diffusion_policy/config/rl_annotate.yaml +265 -0
  75. diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
  76. diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
  77. diffusion_policy/config/rl_annotate_v2.yaml +141 -0
  78. diffusion_policy/config/rl_annotate_v3.yaml +165 -0
  79. diffusion_policy/config/task/lift.yaml +89 -0
  80. diffusion_policy/config/task/square.yaml +92 -0
  81. diffusion_policy/config/task/tool_hang.yaml +93 -0
  82. diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
  83. diffusion_policy/config/task/umi.yaml +105 -0
  84. diffusion_policy/config/task/umi_bimanual.yaml +135 -0
  85. diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
  86. diffusion_policy/config/task/umi_image.yaml +36 -0
  87. diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
  88. diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
  89. diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
  90. diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
  91. diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
  92. diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
  93. diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
  94. diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
  95. diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
  96. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
  97. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
  98. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
  99. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
  100. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
  101. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
  102. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
  103. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
  104. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
  105. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
  106. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
  107. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
  108. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
  109. diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
  110. diffusion_policy/config/train_iql_umi.yaml +186 -0
  111. diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
  112. diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
  113. diffusion_policy/dataset/__init__.py +0 -0
  114. diffusion_policy/dataset/base_dataset.py +76 -0
  115. diffusion_policy/dataset/umi_dataset.py +652 -0
  116. diffusion_policy/env_runner/__init__.py +0 -0
  117. diffusion_policy/env_runner/base_image_runner.py +9 -0
  118. diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
  119. diffusion_policy/model/__init__.py +0 -0
  120. diffusion_policy/model/common/__init__.py +0 -0
  121. diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
  122. diffusion_policy/model/common/lr_scheduler.py +46 -0
  123. diffusion_policy/model/common/module_attr_mixin.py +14 -0
  124. diffusion_policy/model/common/normalizer.py +353 -0
  125. diffusion_policy/model/common/tensor_util.py +960 -0
  126. diffusion_policy/model/diffusion/__init__.py +0 -0
  127. diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
  128. diffusion_policy/model/diffusion/conv1d_components.py +46 -0
  129. diffusion_policy/model/diffusion/ema_model.py +88 -0
  130. diffusion_policy/model/diffusion/mask_generator.py +221 -0
  131. diffusion_policy/model/diffusion/positional_embedding.py +17 -0
  132. diffusion_policy/model/vision/__init__.py +0 -0
  133. diffusion_policy/model/vision/crop_randomizer.py +288 -0
  134. diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
  135. diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
  136. diffusion_policy/policy/__init__.py +0 -0
  137. diffusion_policy/policy/base_image_policy.py +27 -0
  138. diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
  139. diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
  140. diffusion_policy/policy/parl_action_optimizer.py +675 -0
  141. diffusion_policy/workspace/__init__.py +0 -0
  142. diffusion_policy/workspace/base_workspace.py +146 -0
  143. diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
  144. solarian_diffusion/__init__.py +32 -0
  145. solarian_diffusion/config.py +31 -0
  146. solarian_diffusion/policy_server.py +316 -0
  147. solarian_diffusion/rtc.py +272 -0
  148. solarian_diffusion/service.py +16 -0
  149. solarian_diffusion/trainer.py +213 -0
  150. solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
  151. solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
  152. solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
  153. solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
  154. solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
  155. solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
  156. solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
  157. umi/__init__.py +0 -0
  158. umi/common/__init__.py +0 -0
  159. umi/common/pose_util.py +115 -0
@@ -0,0 +1,324 @@
1
+ # =============================================================================
2
+ # BC-anchored AWR finetune of the BC pick_lego_pile_bbox diffusion policy.
3
+ #
4
+ # Recipe (per the "stronger BC anchoring" RL experiment notes): combine
5
+ # the standard advantage-weighted denoising loss with TWO anchor terms
6
+ # back toward the BC checkpoint so the policy doesn't drift off the
7
+ # imitation manifold and produce the twitchy / unstable motions we saw
8
+ # from the previous unconstrained AWR run:
9
+ #
10
+ # total_loss = AWR_term # main reward signal
11
+ # + bc_lambda * unweighted_BC_denoising_loss # "treat every demo equally"
12
+ # + anc_lambda * mse(eps_new, eps_frozen_bc) # soft-KL anchor in eps space
13
+ #
14
+ # The frozen reference U-Net is a deep copy of the BC ckpt's U-Net,
15
+ # snapshotted by the workspace right after `init_from_ckpt` loads
16
+ # (see ``init_frozen_ref()`` on DiffusionUnetTimmPolicy). The anchor
17
+ # term is silently disabled if ``awr_anchor_lambda <= 0`` so legacy
18
+ # AWR runs cost nothing extra in GPU memory.
19
+ #
20
+ # This config is a *finetuning* run: it expects to be initialized from a
21
+ # trained BC checkpoint (`training.init_from_ckpt`) and to be pointed at a
22
+ # zarr.zip that already has `rl/q` and `rl/v` written by `zarr_q_v.py`. The
23
+ # UmiDataset auto-detects those arrays and emits per-sample `advantage` /
24
+ # `has_advantage` keys; DiffusionUnetTimmPolicy reads them only when
25
+ # `policy.awr_beta > 0`.
26
+ #
27
+ # Knobs grouped by what to tune in what order:
28
+ #
29
+ # 1) BASELINE SAFETY KNOBS (already set sensibly here):
30
+ # training.freeze_encoder = true
31
+ # training.init_from_ckpt = <BC ckpt>
32
+ # optimizer.lr = 1.0e-5 (RL paper recipe: 1e-5..5e-5)
33
+ # training.num_epochs = 10
34
+ # ema.update_after_step = 0
35
+ #
36
+ # 2) AWR-SHAPE KNOBS (start with the conservative defaults below; only
37
+ # relax once stable rollouts are confirmed):
38
+ # policy.awr_beta = 5.0 (high temperature -> mild reweighting)
39
+ # policy.awr_w_max = 3.0 (hard cap; tame outliers)
40
+ # policy.awr_w_min = 0.5 (don't let any sample go below half BC)
41
+ # policy.awr_lambda = 1.0 (full AWR term, but with above clamps)
42
+ # policy.awr_normalize_weights = true
43
+ #
44
+ # 3) BC-ANCHOR KNOBS (the new "don't drift" levers):
45
+ # policy.awr_bc_lambda = 1.0 (unweighted BC, free since no extra forward)
46
+ # policy.awr_anchor_lambda = 0.3 (frozen-ref eps-MSE; ~1 extra U-Net fwd/step)
47
+ #
48
+ # Smoke-test recipes:
49
+ # * Reproduce legacy AWR run byte-identically:
50
+ # policy.awr_beta=0.15 policy.awr_w_max=20.0 policy.awr_w_min=0.0
51
+ # policy.awr_bc_lambda=0.0 policy.awr_anchor_lambda=0.0
52
+ # * Reproduce plain BC byte-identically:
53
+ # policy.awr_beta=0.0 policy.awr_lambda=1.0
54
+ # policy.awr_bc_lambda=0.0 policy.awr_anchor_lambda=0.0
55
+ # * "Sanity-bake" with just the anchor (no AWR signal): set awr_beta=0
56
+ # and awr_anchor_lambda=0.3 -- val_loss should remain within ~5% of BC.
57
+ #
58
+ # Note: with `_self_` placed AFTER the task default (Hydra 1.1 idiom), any
59
+ # fields set in this file override the task config's values. The BC sibling
60
+ # config (without `_awr` suffix) keeps `_self_` first because it never
61
+ # overrides task fields; we deviate here on purpose.
62
+ # =============================================================================
63
+
64
+ defaults:
65
+ - task: umi_pick_single_lego_bbox
66
+ - _self_
67
+
68
+ name: train_diffusion_unet_timm_awr
69
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
70
+
71
+ shape_meta: ${task.shape_meta}
72
+ exp_name: "awr"
73
+ batch_size: 128
74
+ visual_encoder: "vit_large_patch14_clip_224"
75
+ visual_encoder_tag: "vitl"
76
+ model_type: "unet"
77
+ num_gpus: 1
78
+
79
+ # Same parse helpers as the BC config so wandb tags resolve identically.
80
+ mirror: ${parse_dataset_field:${task.dataset_path},mirror,no_mirror}
81
+ real_data_size: ${parse_dataset_field:${task.dataset_path},real_data_size,null}
82
+ sim_data_size: ${parse_dataset_field:${task.dataset_path},sim_data_size,null}
83
+ bbox: ${parse_dataset_field:${task.dataset_path},bbox,no_bbox}
84
+ bbox_mode: ${task.shape_meta.obs.bbox.bbox_mode}
85
+ task_name: ${parse_dataset_field:${task.dataset_path},data_task,pick_lego}
86
+
87
+ n_action_steps: 8
88
+
89
+ policy:
90
+ _target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
91
+
92
+ shape_meta: ${shape_meta}
93
+
94
+ noise_scheduler:
95
+ _target_: diffusers.DDIMScheduler
96
+ num_train_timesteps: 50
97
+ beta_start: 0.0001
98
+ beta_end: 0.02
99
+ beta_schedule: squaredcos_cap_v2
100
+ clip_sample: True
101
+ set_alpha_to_one: True
102
+ steps_offset: 0
103
+ prediction_type: epsilon
104
+
105
+ obs_encoder:
106
+ _target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
107
+ shape_meta: ${shape_meta}
108
+ model_name: ${visual_encoder}
109
+ pretrained: True
110
+ # frozen=False keeps the encoder trainable in principle; the actual
111
+ # freeze for AWR finetuning happens via `training.freeze_encoder`
112
+ # below, which sets requires_grad_(False) every epoch. Leaving
113
+ # frozen=False here matches the BC config so the LR multiplier
114
+ # logic in the workspace keeps working.
115
+ frozen: False
116
+
117
+ global_pool: ''
118
+ feature_aggregation: 'attention_pool_2d'
119
+ position_encording: 'sinusoidal'
120
+ downsample_ratio: 32
121
+
122
+ # AWR runs reweight existing samples; they already have a much
123
+ # smaller *effective* dataset than BC. Heavy augmentation here
124
+ # would dilute the high-advantage signal, so we keep ColorJitter
125
+ # but drop RandomCrop's spatial randomness. Re-enable the crop if
126
+ # you find the model overfitting (``train_loss`` falling fast
127
+ # while ``val_loss`` stays flat).
128
+ transforms:
129
+ - type: RandomCrop
130
+ ratio: 0.95
131
+ - _target_: torchvision.transforms.ColorJitter
132
+ brightness: 0.3
133
+ contrast: 0.4
134
+ saturation: 0.5
135
+ hue: 0.08
136
+
137
+ use_group_norm: True
138
+ share_rgb_model: False
139
+ imagenet_norm: True
140
+
141
+ num_inference_steps: 16
142
+ obs_as_global_cond: True
143
+ diffusion_step_embed_dim: 128
144
+ down_dims: [256,512,1024]
145
+ kernel_size: 5
146
+ n_groups: 8
147
+ cond_predict_scale: True
148
+ input_pertub: 0.1
149
+ train_diffusion_n_samples: 1
150
+
151
+ # ============== AWR knobs (BC-anchored recipe) ==============
152
+ # Temperature on the advantage. We set it MUCH higher than the
153
+ # original AWR run (was 0.15) so the per-sample weight distribution
154
+ # is mild -- preventing the optimizer from over-fitting to the
155
+ # critic's biggest-advantage outliers. With the conservative
156
+ # awr_w_max=3 cap below, the resulting weights span roughly
157
+ # [0.5, 3.0] -- a 6x dynamic range instead of the 130x the legacy
158
+ # config could produce. Drop beta toward 1-2 only after stable
159
+ # rollouts confirm the critic isn't being exploited.
160
+ awr_beta: 5.0
161
+ # Hard upper cap on per-sample weight pre-normalization. Combined
162
+ # with awr_beta=5 this means the effective weight ratio between the
163
+ # best- and worst-advantage samples is at most ~6x. The legacy
164
+ # default of 20 frequently let single outliers dominate a batch.
165
+ awr_w_max: 3.0
166
+ # Lower floor on per-sample weight (NEW). 0.5 means every sample
167
+ # carries at least half the BC weight regardless of advantage, so
168
+ # low-advantage but on-distribution samples can't be silently
169
+ # zeroed by a very low (or negative) `adv / beta`. Set to 0.0 to
170
+ # disable for legacy-AWR-equivalent behavior.
171
+ awr_w_min: 0.5
172
+ # 1.0 = pure AWR. Set to 0.5 (or lower) as a "safety belt": the loss
173
+ # becomes a 50/50 average of plain BC and AWR-weighted BC, which
174
+ # bounds the policy drift if the critic turns out to be miscalibrated.
175
+ # 0.0 = pure BC (smoke-test mode -- should reproduce the legacy run).
176
+ awr_lambda: 1.0
177
+ # Per-batch w / w.mean() normalization. Strongly recommended: keeps
178
+ # the effective LR invariant to beta and to which subset of the
179
+ # batch happens to have high advantage.
180
+ awr_normalize_weights: true
181
+ # ===== BC anchor terms (NEW) =====
182
+ # Coefficient on the UNWEIGHTED denoising loss (plain BC). Cheap:
183
+ # uses the same per-sample MSE the AWR term already computed, no
184
+ # extra forward pass. 0.0 disables; 1.0 means the total loss is
185
+ # `AWR + BC` so the policy is pulled back toward "imitate every
186
+ # demo equally" with equal weight to the advantage signal.
187
+ awr_bc_lambda: 1.0
188
+ # Coefficient on the frozen-reference anchor (epsilon-space MSE
189
+ # against the BC ckpt's U-Net forward at the SAME noisy_trajectory
190
+ # / timesteps / global_cond). Acts as a soft KL toward the BC
191
+ # policy in epsilon-space. Costs ONE extra U-Net forward per step
192
+ # (no backward, ref is frozen). Set to 0.0 to skip the anchor and
193
+ # save ~600MB of GPU memory.
194
+ awr_anchor_lambda: 0.3
195
+
196
+ ema:
197
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
198
+ update_after_step: 0
199
+ inv_gamma: 1.0
200
+ power: 0.75
201
+ min_value: 0.0
202
+ max_value: 0.9999
203
+
204
+ dataloader:
205
+ batch_size: ${batch_size}
206
+ num_workers: 8
207
+ shuffle: True
208
+ pin_memory: True
209
+ persistent_workers: True
210
+ prefetch_factor: 4
211
+
212
+ val_dataloader:
213
+ batch_size: ${batch_size}
214
+ num_workers: 8
215
+ shuffle: False
216
+ pin_memory: True
217
+ persistent_workers: True
218
+ prefetch_factor: 4
219
+
220
+ optimizer:
221
+ _target_: torch.optim.AdamW
222
+ # 1e-5 -- the RL-fine-tune recipe's recommended range is 1e-5..5e-5
223
+ # for BC-anchored AWR. The legacy AWR config used 1e-6 because it
224
+ # had NO anchor and was therefore extremely sensitive to drift;
225
+ # with the bc_lambda + anchor_lambda terms above bounding policy
226
+ # drift, we can move ~10x faster without destabilizing rollouts.
227
+ lr: 1.0e-5
228
+ betas: [0.95, 0.999]
229
+ eps: 1.0e-8
230
+ weight_decay: 1.0e-6
231
+
232
+ training:
233
+ device: "cuda:0"
234
+ seed: 42
235
+ debug: False
236
+ resume: False
237
+ lr_scheduler: cosine
238
+ # Cosine + 100-step warmup. The schedule is short (10 epochs) so
239
+ # warmup needs to be too -- a long warmup would never reach the
240
+ # peak LR before training ends.
241
+ lr_warmup_steps: 100
242
+ # Short on purpose. AWR re-weights an existing dataset; once the
243
+ # high-advantage samples are absorbed there's little to learn and
244
+ # over-training degrades visual generalization. Bump to 30 if you
245
+ # see val_loss / action_mse still trending down at epoch 10.
246
+ num_epochs: 10
247
+ gradient_accumulate_every: 1
248
+ use_ema: True
249
+ # Freeze the visual encoder for AWR. The features that worked for
250
+ # BC are the same features the IQL critic was trained against; if
251
+ # we let them drift, the critic's Q estimate becomes uncalibrated
252
+ # and the AWR weights point in the wrong direction.
253
+ freeze_encoder: True
254
+
255
+ # Path to the BC checkpoint to initialize from. SET THIS BEFORE
256
+ # RUNNING. Optimizer state, global_step, and epoch counters are
257
+ # NOT loaded -- the run starts from step 0 with a fresh AdamW.
258
+ # Override on CLI: training.init_from_ckpt=/path/to/bc.ckpt
259
+ init_from_ckpt: null
260
+
261
+ rollout_every: 10
262
+ checkpoint_every: 1
263
+ val_every: 1
264
+ sample_every: 1
265
+ max_train_steps: null
266
+ max_val_steps: null
267
+ tqdm_interval_sec: 1.0
268
+
269
+ logging:
270
+ project: umi
271
+ resume: False
272
+ mode: online
273
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
274
+ tags:
275
+ - "${name}"
276
+ - "${task_name}"
277
+ - "${exp_name}"
278
+ - "awr"
279
+ - "${visual_encoder_tag}"
280
+ - "batch_size:${eval:'${batch_size} * ${num_gpus}'}"
281
+ - "num_gpus:${num_gpus}"
282
+ - "${model_type}"
283
+ - "${eval:'\"${task.dataset_path}\".split(\"/\")[-1].rsplit(\".\",1)[0]'}"
284
+ - "${mirror}"
285
+ - "${eval:'\"real_size:%d\" % ${real_data_size} if ${real_data_size} is not None else \"\"'}"
286
+ - "${eval:'\"sim_size:%d\" % ${sim_data_size} if ${sim_data_size} is not None else \"\"'}"
287
+ - "${eval:'(\"real\" if ${real_data_size} > 0 else \"no_real\") if ${real_data_size} is not None else \"\"'}"
288
+ - "${eval:'(\"sim\" if ${sim_data_size} > 0 else \"no_sim\") if ${sim_data_size} is not None else \"\"'}"
289
+ - "${bbox}"
290
+ - "bbox_mode:${bbox_mode}"
291
+ - "awr_beta:${policy.awr_beta}"
292
+ - "awr_lambda:${policy.awr_lambda}"
293
+ - "awr_w_min:${policy.awr_w_min}"
294
+ - "awr_w_max:${policy.awr_w_max}"
295
+ - "awr_bc_lambda:${policy.awr_bc_lambda}"
296
+ - "awr_anchor_lambda:${policy.awr_anchor_lambda}"
297
+ id: null
298
+ group: ${task_name}_${model_type}_awr
299
+
300
+ checkpoint:
301
+ topk:
302
+ # Track val_loss instead of train_loss for AWR finetuning.
303
+ # train_loss is dominated by the AWR weight magnitude and is
304
+ # NOT comparable across awr_beta values; val_loss is plain
305
+ # unweighted MSE so it's the right thing to rank ckpts on.
306
+ monitor_key: val_loss
307
+ mode: min
308
+ k: 10
309
+ format_str: '{task_name}-awr-{model_type}-{visual_encoder}-batch_{total_batch}-{dataset_file_name}-epoch_{epoch:04d}-val_loss_{val_loss:.4f}.ckpt'
310
+ save_last_ckpt: True
311
+ save_last_snapshot: False
312
+
313
+ multi_run:
314
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
315
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
316
+
317
+ hydra:
318
+ job:
319
+ override_dirname: ${name}
320
+ run:
321
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
322
+ sweep:
323
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
324
+ subdir: ${hydra.job.num}