solarian-diffusion 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffusion_policy/__init__.py +0 -0
- diffusion_policy/codecs/__init__.py +0 -0
- diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
- diffusion_policy/common/__init__.py +0 -0
- diffusion_policy/common/checkpoint_util.py +64 -0
- diffusion_policy/common/json_logger.py +117 -0
- diffusion_policy/common/normalize_util.py +251 -0
- diffusion_policy/common/pose_repr_util.py +112 -0
- diffusion_policy/common/pytorch_util.py +82 -0
- diffusion_policy/common/q_critic_util.py +135 -0
- diffusion_policy/common/replay_buffer.py +672 -0
- diffusion_policy/common/sampler.py +204 -0
- diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
- diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
- diffusion_policy/config/legacy/task/can_image.yaml +64 -0
- diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
- diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
- diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
- diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
- diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
- diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
- diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
- diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
- diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
- diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
- diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
- diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
- diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
- diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
- diffusion_policy/config/legacy/task/square_image.yaml +64 -0
- diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
- diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
- diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
- diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
- diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
- diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
- diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
- diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
- diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
- diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
- diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
- diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
- diffusion_policy/config/legacy/task/umi.yaml +45 -0
- diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
- diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
- diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
- diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
- diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
- diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
- diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
- diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
- diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
- diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
- diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
- diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
- diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
- diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
- diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
- diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
- diffusion_policy/config/rl_annotate.yaml +265 -0
- diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
- diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
- diffusion_policy/config/rl_annotate_v2.yaml +141 -0
- diffusion_policy/config/rl_annotate_v3.yaml +165 -0
- diffusion_policy/config/task/lift.yaml +89 -0
- diffusion_policy/config/task/square.yaml +92 -0
- diffusion_policy/config/task/tool_hang.yaml +93 -0
- diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
- diffusion_policy/config/task/umi.yaml +105 -0
- diffusion_policy/config/task/umi_bimanual.yaml +135 -0
- diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
- diffusion_policy/config/task/umi_image.yaml +36 -0
- diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
- diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
- diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
- diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
- diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
- diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
- diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
- diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
- diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
- diffusion_policy/config/train_iql_umi.yaml +186 -0
- diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
- diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
- diffusion_policy/dataset/__init__.py +0 -0
- diffusion_policy/dataset/base_dataset.py +76 -0
- diffusion_policy/dataset/umi_dataset.py +652 -0
- diffusion_policy/env_runner/__init__.py +0 -0
- diffusion_policy/env_runner/base_image_runner.py +9 -0
- diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
- diffusion_policy/model/__init__.py +0 -0
- diffusion_policy/model/common/__init__.py +0 -0
- diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
- diffusion_policy/model/common/lr_scheduler.py +46 -0
- diffusion_policy/model/common/module_attr_mixin.py +14 -0
- diffusion_policy/model/common/normalizer.py +353 -0
- diffusion_policy/model/common/tensor_util.py +960 -0
- diffusion_policy/model/diffusion/__init__.py +0 -0
- diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
- diffusion_policy/model/diffusion/conv1d_components.py +46 -0
- diffusion_policy/model/diffusion/ema_model.py +88 -0
- diffusion_policy/model/diffusion/mask_generator.py +221 -0
- diffusion_policy/model/diffusion/positional_embedding.py +17 -0
- diffusion_policy/model/vision/__init__.py +0 -0
- diffusion_policy/model/vision/crop_randomizer.py +288 -0
- diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
- diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
- diffusion_policy/policy/__init__.py +0 -0
- diffusion_policy/policy/base_image_policy.py +27 -0
- diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
- diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
- diffusion_policy/policy/parl_action_optimizer.py +675 -0
- diffusion_policy/workspace/__init__.py +0 -0
- diffusion_policy/workspace/base_workspace.py +146 -0
- diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
- solarian_diffusion/__init__.py +32 -0
- solarian_diffusion/config.py +31 -0
- solarian_diffusion/policy_server.py +316 -0
- solarian_diffusion/rtc.py +272 -0
- solarian_diffusion/service.py +16 -0
- solarian_diffusion/trainer.py +213 -0
- solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
- solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
- solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
- solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
- solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
- solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
- solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
- umi/__init__.py +0 -0
- umi/common/__init__.py +0 -0
- umi/common/pose_util.py +115 -0
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
# =============================================================================
|
|
2
|
+
# BC-anchored AWR finetune of the BC pick_lego_pile_bbox diffusion policy.
|
|
3
|
+
#
|
|
4
|
+
# Recipe (per the "stronger BC anchoring" RL experiment notes): combine
|
|
5
|
+
# the standard advantage-weighted denoising loss with TWO anchor terms
|
|
6
|
+
# back toward the BC checkpoint so the policy doesn't drift off the
|
|
7
|
+
# imitation manifold and produce the twitchy / unstable motions we saw
|
|
8
|
+
# from the previous unconstrained AWR run:
|
|
9
|
+
#
|
|
10
|
+
# total_loss = AWR_term # main reward signal
|
|
11
|
+
# + bc_lambda * unweighted_BC_denoising_loss # "treat every demo equally"
|
|
12
|
+
# + anc_lambda * mse(eps_new, eps_frozen_bc) # soft-KL anchor in eps space
|
|
13
|
+
#
|
|
14
|
+
# The frozen reference U-Net is a deep copy of the BC ckpt's U-Net,
|
|
15
|
+
# snapshotted by the workspace right after `init_from_ckpt` loads
|
|
16
|
+
# (see ``init_frozen_ref()`` on DiffusionUnetTimmPolicy). The anchor
|
|
17
|
+
# term is silently disabled if ``awr_anchor_lambda <= 0`` so legacy
|
|
18
|
+
# AWR runs cost nothing extra in GPU memory.
|
|
19
|
+
#
|
|
20
|
+
# This config is a *finetuning* run: it expects to be initialized from a
|
|
21
|
+
# trained BC checkpoint (`training.init_from_ckpt`) and to be pointed at a
|
|
22
|
+
# zarr.zip that already has `rl/q` and `rl/v` written by `zarr_q_v.py`. The
|
|
23
|
+
# UmiDataset auto-detects those arrays and emits per-sample `advantage` /
|
|
24
|
+
# `has_advantage` keys; DiffusionUnetTimmPolicy reads them only when
|
|
25
|
+
# `policy.awr_beta > 0`.
|
|
26
|
+
#
|
|
27
|
+
# Knobs grouped by what to tune in what order:
|
|
28
|
+
#
|
|
29
|
+
# 1) BASELINE SAFETY KNOBS (already set sensibly here):
|
|
30
|
+
# training.freeze_encoder = true
|
|
31
|
+
# training.init_from_ckpt = <BC ckpt>
|
|
32
|
+
# optimizer.lr = 1.0e-5 (RL paper recipe: 1e-5..5e-5)
|
|
33
|
+
# training.num_epochs = 10
|
|
34
|
+
# ema.update_after_step = 0
|
|
35
|
+
#
|
|
36
|
+
# 2) AWR-SHAPE KNOBS (start with the conservative defaults below; only
|
|
37
|
+
# relax once stable rollouts are confirmed):
|
|
38
|
+
# policy.awr_beta = 5.0 (high temperature -> mild reweighting)
|
|
39
|
+
# policy.awr_w_max = 3.0 (hard cap; tame outliers)
|
|
40
|
+
# policy.awr_w_min = 0.5 (don't let any sample go below half BC)
|
|
41
|
+
# policy.awr_lambda = 1.0 (full AWR term, but with above clamps)
|
|
42
|
+
# policy.awr_normalize_weights = true
|
|
43
|
+
#
|
|
44
|
+
# 3) BC-ANCHOR KNOBS (the new "don't drift" levers):
|
|
45
|
+
# policy.awr_bc_lambda = 1.0 (unweighted BC, free since no extra forward)
|
|
46
|
+
# policy.awr_anchor_lambda = 0.3 (frozen-ref eps-MSE; ~1 extra U-Net fwd/step)
|
|
47
|
+
#
|
|
48
|
+
# Smoke-test recipes:
|
|
49
|
+
# * Reproduce legacy AWR run byte-identically:
|
|
50
|
+
# policy.awr_beta=0.15 policy.awr_w_max=20.0 policy.awr_w_min=0.0
|
|
51
|
+
# policy.awr_bc_lambda=0.0 policy.awr_anchor_lambda=0.0
|
|
52
|
+
# * Reproduce plain BC byte-identically:
|
|
53
|
+
# policy.awr_beta=0.0 policy.awr_lambda=1.0
|
|
54
|
+
# policy.awr_bc_lambda=0.0 policy.awr_anchor_lambda=0.0
|
|
55
|
+
# * "Sanity-bake" with just the anchor (no AWR signal): set awr_beta=0
|
|
56
|
+
# and awr_anchor_lambda=0.3 -- val_loss should remain within ~5% of BC.
|
|
57
|
+
#
|
|
58
|
+
# Note: with `_self_` placed AFTER the task default (Hydra 1.1 idiom), any
|
|
59
|
+
# fields set in this file override the task config's values. The BC sibling
|
|
60
|
+
# config (without `_awr` suffix) keeps `_self_` first because it never
|
|
61
|
+
# overrides task fields; we deviate here on purpose.
|
|
62
|
+
# =============================================================================
|
|
63
|
+
|
|
64
|
+
defaults:
|
|
65
|
+
- task: umi_pick_single_lego_bbox
|
|
66
|
+
- _self_
|
|
67
|
+
|
|
68
|
+
name: train_diffusion_unet_timm_awr
|
|
69
|
+
_target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
|
|
70
|
+
|
|
71
|
+
shape_meta: ${task.shape_meta}
|
|
72
|
+
exp_name: "awr"
|
|
73
|
+
batch_size: 128
|
|
74
|
+
visual_encoder: "vit_large_patch14_clip_224"
|
|
75
|
+
visual_encoder_tag: "vitl"
|
|
76
|
+
model_type: "unet"
|
|
77
|
+
num_gpus: 1
|
|
78
|
+
|
|
79
|
+
# Same parse helpers as the BC config so wandb tags resolve identically.
|
|
80
|
+
mirror: ${parse_dataset_field:${task.dataset_path},mirror,no_mirror}
|
|
81
|
+
real_data_size: ${parse_dataset_field:${task.dataset_path},real_data_size,null}
|
|
82
|
+
sim_data_size: ${parse_dataset_field:${task.dataset_path},sim_data_size,null}
|
|
83
|
+
bbox: ${parse_dataset_field:${task.dataset_path},bbox,no_bbox}
|
|
84
|
+
bbox_mode: ${task.shape_meta.obs.bbox.bbox_mode}
|
|
85
|
+
task_name: ${parse_dataset_field:${task.dataset_path},data_task,pick_lego}
|
|
86
|
+
|
|
87
|
+
n_action_steps: 8
|
|
88
|
+
|
|
89
|
+
policy:
|
|
90
|
+
_target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
|
|
91
|
+
|
|
92
|
+
shape_meta: ${shape_meta}
|
|
93
|
+
|
|
94
|
+
noise_scheduler:
|
|
95
|
+
_target_: diffusers.DDIMScheduler
|
|
96
|
+
num_train_timesteps: 50
|
|
97
|
+
beta_start: 0.0001
|
|
98
|
+
beta_end: 0.02
|
|
99
|
+
beta_schedule: squaredcos_cap_v2
|
|
100
|
+
clip_sample: True
|
|
101
|
+
set_alpha_to_one: True
|
|
102
|
+
steps_offset: 0
|
|
103
|
+
prediction_type: epsilon
|
|
104
|
+
|
|
105
|
+
obs_encoder:
|
|
106
|
+
_target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
|
|
107
|
+
shape_meta: ${shape_meta}
|
|
108
|
+
model_name: ${visual_encoder}
|
|
109
|
+
pretrained: True
|
|
110
|
+
# frozen=False keeps the encoder trainable in principle; the actual
|
|
111
|
+
# freeze for AWR finetuning happens via `training.freeze_encoder`
|
|
112
|
+
# below, which sets requires_grad_(False) every epoch. Leaving
|
|
113
|
+
# frozen=False here matches the BC config so the LR multiplier
|
|
114
|
+
# logic in the workspace keeps working.
|
|
115
|
+
frozen: False
|
|
116
|
+
|
|
117
|
+
global_pool: ''
|
|
118
|
+
feature_aggregation: 'attention_pool_2d'
|
|
119
|
+
position_encording: 'sinusoidal'
|
|
120
|
+
downsample_ratio: 32
|
|
121
|
+
|
|
122
|
+
# AWR runs reweight existing samples; they already have a much
|
|
123
|
+
# smaller *effective* dataset than BC. Heavy augmentation here
|
|
124
|
+
# would dilute the high-advantage signal, so we keep ColorJitter
|
|
125
|
+
# but drop RandomCrop's spatial randomness. Re-enable the crop if
|
|
126
|
+
# you find the model overfitting (``train_loss`` falling fast
|
|
127
|
+
# while ``val_loss`` stays flat).
|
|
128
|
+
transforms:
|
|
129
|
+
- type: RandomCrop
|
|
130
|
+
ratio: 0.95
|
|
131
|
+
- _target_: torchvision.transforms.ColorJitter
|
|
132
|
+
brightness: 0.3
|
|
133
|
+
contrast: 0.4
|
|
134
|
+
saturation: 0.5
|
|
135
|
+
hue: 0.08
|
|
136
|
+
|
|
137
|
+
use_group_norm: True
|
|
138
|
+
share_rgb_model: False
|
|
139
|
+
imagenet_norm: True
|
|
140
|
+
|
|
141
|
+
num_inference_steps: 16
|
|
142
|
+
obs_as_global_cond: True
|
|
143
|
+
diffusion_step_embed_dim: 128
|
|
144
|
+
down_dims: [256,512,1024]
|
|
145
|
+
kernel_size: 5
|
|
146
|
+
n_groups: 8
|
|
147
|
+
cond_predict_scale: True
|
|
148
|
+
input_pertub: 0.1
|
|
149
|
+
train_diffusion_n_samples: 1
|
|
150
|
+
|
|
151
|
+
# ============== AWR knobs (BC-anchored recipe) ==============
|
|
152
|
+
# Temperature on the advantage. We set it MUCH higher than the
|
|
153
|
+
# original AWR run (was 0.15) so the per-sample weight distribution
|
|
154
|
+
# is mild -- preventing the optimizer from over-fitting to the
|
|
155
|
+
# critic's biggest-advantage outliers. With the conservative
|
|
156
|
+
# awr_w_max=3 cap below, the resulting weights span roughly
|
|
157
|
+
# [0.5, 3.0] -- a 6x dynamic range instead of the 130x the legacy
|
|
158
|
+
# config could produce. Drop beta toward 1-2 only after stable
|
|
159
|
+
# rollouts confirm the critic isn't being exploited.
|
|
160
|
+
awr_beta: 5.0
|
|
161
|
+
# Hard upper cap on per-sample weight pre-normalization. Combined
|
|
162
|
+
# with awr_beta=5 this means the effective weight ratio between the
|
|
163
|
+
# best- and worst-advantage samples is at most ~6x. The legacy
|
|
164
|
+
# default of 20 frequently let single outliers dominate a batch.
|
|
165
|
+
awr_w_max: 3.0
|
|
166
|
+
# Lower floor on per-sample weight (NEW). 0.5 means every sample
|
|
167
|
+
# carries at least half the BC weight regardless of advantage, so
|
|
168
|
+
# low-advantage but on-distribution samples can't be silently
|
|
169
|
+
# zeroed by a very low (or negative) `adv / beta`. Set to 0.0 to
|
|
170
|
+
# disable for legacy-AWR-equivalent behavior.
|
|
171
|
+
awr_w_min: 0.5
|
|
172
|
+
# 1.0 = pure AWR. Set to 0.5 (or lower) as a "safety belt": the loss
|
|
173
|
+
# becomes a 50/50 average of plain BC and AWR-weighted BC, which
|
|
174
|
+
# bounds the policy drift if the critic turns out to be miscalibrated.
|
|
175
|
+
# 0.0 = pure BC (smoke-test mode -- should reproduce the legacy run).
|
|
176
|
+
awr_lambda: 1.0
|
|
177
|
+
# Per-batch w / w.mean() normalization. Strongly recommended: keeps
|
|
178
|
+
# the effective LR invariant to beta and to which subset of the
|
|
179
|
+
# batch happens to have high advantage.
|
|
180
|
+
awr_normalize_weights: true
|
|
181
|
+
# ===== BC anchor terms (NEW) =====
|
|
182
|
+
# Coefficient on the UNWEIGHTED denoising loss (plain BC). Cheap:
|
|
183
|
+
# uses the same per-sample MSE the AWR term already computed, no
|
|
184
|
+
# extra forward pass. 0.0 disables; 1.0 means the total loss is
|
|
185
|
+
# `AWR + BC` so the policy is pulled back toward "imitate every
|
|
186
|
+
# demo equally" with equal weight to the advantage signal.
|
|
187
|
+
awr_bc_lambda: 1.0
|
|
188
|
+
# Coefficient on the frozen-reference anchor (epsilon-space MSE
|
|
189
|
+
# against the BC ckpt's U-Net forward at the SAME noisy_trajectory
|
|
190
|
+
# / timesteps / global_cond). Acts as a soft KL toward the BC
|
|
191
|
+
# policy in epsilon-space. Costs ONE extra U-Net forward per step
|
|
192
|
+
# (no backward, ref is frozen). Set to 0.0 to skip the anchor and
|
|
193
|
+
# save ~600MB of GPU memory.
|
|
194
|
+
awr_anchor_lambda: 0.3
|
|
195
|
+
|
|
196
|
+
ema:
|
|
197
|
+
_target_: diffusion_policy.model.diffusion.ema_model.EMAModel
|
|
198
|
+
update_after_step: 0
|
|
199
|
+
inv_gamma: 1.0
|
|
200
|
+
power: 0.75
|
|
201
|
+
min_value: 0.0
|
|
202
|
+
max_value: 0.9999
|
|
203
|
+
|
|
204
|
+
dataloader:
|
|
205
|
+
batch_size: ${batch_size}
|
|
206
|
+
num_workers: 8
|
|
207
|
+
shuffle: True
|
|
208
|
+
pin_memory: True
|
|
209
|
+
persistent_workers: True
|
|
210
|
+
prefetch_factor: 4
|
|
211
|
+
|
|
212
|
+
val_dataloader:
|
|
213
|
+
batch_size: ${batch_size}
|
|
214
|
+
num_workers: 8
|
|
215
|
+
shuffle: False
|
|
216
|
+
pin_memory: True
|
|
217
|
+
persistent_workers: True
|
|
218
|
+
prefetch_factor: 4
|
|
219
|
+
|
|
220
|
+
optimizer:
|
|
221
|
+
_target_: torch.optim.AdamW
|
|
222
|
+
# 1e-5 -- the RL-fine-tune recipe's recommended range is 1e-5..5e-5
|
|
223
|
+
# for BC-anchored AWR. The legacy AWR config used 1e-6 because it
|
|
224
|
+
# had NO anchor and was therefore extremely sensitive to drift;
|
|
225
|
+
# with the bc_lambda + anchor_lambda terms above bounding policy
|
|
226
|
+
# drift, we can move ~10x faster without destabilizing rollouts.
|
|
227
|
+
lr: 1.0e-5
|
|
228
|
+
betas: [0.95, 0.999]
|
|
229
|
+
eps: 1.0e-8
|
|
230
|
+
weight_decay: 1.0e-6
|
|
231
|
+
|
|
232
|
+
training:
|
|
233
|
+
device: "cuda:0"
|
|
234
|
+
seed: 42
|
|
235
|
+
debug: False
|
|
236
|
+
resume: False
|
|
237
|
+
lr_scheduler: cosine
|
|
238
|
+
# Cosine + 100-step warmup. The schedule is short (10 epochs) so
|
|
239
|
+
# warmup needs to be too -- a long warmup would never reach the
|
|
240
|
+
# peak LR before training ends.
|
|
241
|
+
lr_warmup_steps: 100
|
|
242
|
+
# Short on purpose. AWR re-weights an existing dataset; once the
|
|
243
|
+
# high-advantage samples are absorbed there's little to learn and
|
|
244
|
+
# over-training degrades visual generalization. Bump to 30 if you
|
|
245
|
+
# see val_loss / action_mse still trending down at epoch 10.
|
|
246
|
+
num_epochs: 10
|
|
247
|
+
gradient_accumulate_every: 1
|
|
248
|
+
use_ema: True
|
|
249
|
+
# Freeze the visual encoder for AWR. The features that worked for
|
|
250
|
+
# BC are the same features the IQL critic was trained against; if
|
|
251
|
+
# we let them drift, the critic's Q estimate becomes uncalibrated
|
|
252
|
+
# and the AWR weights point in the wrong direction.
|
|
253
|
+
freeze_encoder: True
|
|
254
|
+
|
|
255
|
+
# Path to the BC checkpoint to initialize from. SET THIS BEFORE
|
|
256
|
+
# RUNNING. Optimizer state, global_step, and epoch counters are
|
|
257
|
+
# NOT loaded -- the run starts from step 0 with a fresh AdamW.
|
|
258
|
+
# Override on CLI: training.init_from_ckpt=/path/to/bc.ckpt
|
|
259
|
+
init_from_ckpt: null
|
|
260
|
+
|
|
261
|
+
rollout_every: 10
|
|
262
|
+
checkpoint_every: 1
|
|
263
|
+
val_every: 1
|
|
264
|
+
sample_every: 1
|
|
265
|
+
max_train_steps: null
|
|
266
|
+
max_val_steps: null
|
|
267
|
+
tqdm_interval_sec: 1.0
|
|
268
|
+
|
|
269
|
+
logging:
|
|
270
|
+
project: umi
|
|
271
|
+
resume: False
|
|
272
|
+
mode: online
|
|
273
|
+
name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
|
|
274
|
+
tags:
|
|
275
|
+
- "${name}"
|
|
276
|
+
- "${task_name}"
|
|
277
|
+
- "${exp_name}"
|
|
278
|
+
- "awr"
|
|
279
|
+
- "${visual_encoder_tag}"
|
|
280
|
+
- "batch_size:${eval:'${batch_size} * ${num_gpus}'}"
|
|
281
|
+
- "num_gpus:${num_gpus}"
|
|
282
|
+
- "${model_type}"
|
|
283
|
+
- "${eval:'\"${task.dataset_path}\".split(\"/\")[-1].rsplit(\".\",1)[0]'}"
|
|
284
|
+
- "${mirror}"
|
|
285
|
+
- "${eval:'\"real_size:%d\" % ${real_data_size} if ${real_data_size} is not None else \"\"'}"
|
|
286
|
+
- "${eval:'\"sim_size:%d\" % ${sim_data_size} if ${sim_data_size} is not None else \"\"'}"
|
|
287
|
+
- "${eval:'(\"real\" if ${real_data_size} > 0 else \"no_real\") if ${real_data_size} is not None else \"\"'}"
|
|
288
|
+
- "${eval:'(\"sim\" if ${sim_data_size} > 0 else \"no_sim\") if ${sim_data_size} is not None else \"\"'}"
|
|
289
|
+
- "${bbox}"
|
|
290
|
+
- "bbox_mode:${bbox_mode}"
|
|
291
|
+
- "awr_beta:${policy.awr_beta}"
|
|
292
|
+
- "awr_lambda:${policy.awr_lambda}"
|
|
293
|
+
- "awr_w_min:${policy.awr_w_min}"
|
|
294
|
+
- "awr_w_max:${policy.awr_w_max}"
|
|
295
|
+
- "awr_bc_lambda:${policy.awr_bc_lambda}"
|
|
296
|
+
- "awr_anchor_lambda:${policy.awr_anchor_lambda}"
|
|
297
|
+
id: null
|
|
298
|
+
group: ${task_name}_${model_type}_awr
|
|
299
|
+
|
|
300
|
+
checkpoint:
|
|
301
|
+
topk:
|
|
302
|
+
# Track val_loss instead of train_loss for AWR finetuning.
|
|
303
|
+
# train_loss is dominated by the AWR weight magnitude and is
|
|
304
|
+
# NOT comparable across awr_beta values; val_loss is plain
|
|
305
|
+
# unweighted MSE so it's the right thing to rank ckpts on.
|
|
306
|
+
monitor_key: val_loss
|
|
307
|
+
mode: min
|
|
308
|
+
k: 10
|
|
309
|
+
format_str: '{task_name}-awr-{model_type}-{visual_encoder}-batch_{total_batch}-{dataset_file_name}-epoch_{epoch:04d}-val_loss_{val_loss:.4f}.ckpt'
|
|
310
|
+
save_last_ckpt: True
|
|
311
|
+
save_last_snapshot: False
|
|
312
|
+
|
|
313
|
+
multi_run:
|
|
314
|
+
run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
|
|
315
|
+
wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
|
|
316
|
+
|
|
317
|
+
hydra:
|
|
318
|
+
job:
|
|
319
|
+
override_dirname: ${name}
|
|
320
|
+
run:
|
|
321
|
+
dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
|
|
322
|
+
sweep:
|
|
323
|
+
dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
|
|
324
|
+
subdir: ${hydra.job.num}
|