solarian-diffusion 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. diffusion_policy/__init__.py +0 -0
  2. diffusion_policy/codecs/__init__.py +0 -0
  3. diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
  4. diffusion_policy/common/__init__.py +0 -0
  5. diffusion_policy/common/checkpoint_util.py +64 -0
  6. diffusion_policy/common/json_logger.py +117 -0
  7. diffusion_policy/common/normalize_util.py +251 -0
  8. diffusion_policy/common/pose_repr_util.py +112 -0
  9. diffusion_policy/common/pytorch_util.py +82 -0
  10. diffusion_policy/common/q_critic_util.py +135 -0
  11. diffusion_policy/common/replay_buffer.py +672 -0
  12. diffusion_policy/common/sampler.py +204 -0
  13. diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
  14. diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
  15. diffusion_policy/config/legacy/task/can_image.yaml +64 -0
  16. diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
  17. diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
  18. diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
  19. diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
  20. diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
  21. diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
  22. diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
  23. diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
  24. diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
  25. diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
  26. diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
  27. diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
  28. diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
  29. diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
  30. diffusion_policy/config/legacy/task/square_image.yaml +64 -0
  31. diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
  32. diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
  33. diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
  34. diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
  35. diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
  36. diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
  37. diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
  38. diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
  39. diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
  40. diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
  41. diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
  42. diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
  43. diffusion_policy/config/legacy/task/umi.yaml +45 -0
  44. diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
  45. diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
  46. diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
  47. diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
  48. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
  49. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
  50. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
  51. diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
  52. diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
  53. diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
  54. diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
  55. diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
  56. diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
  57. diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
  58. diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
  59. diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
  60. diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
  61. diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
  62. diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
  63. diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
  64. diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
  65. diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
  66. diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
  67. diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
  68. diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
  69. diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
  70. diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
  71. diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
  72. diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
  73. diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
  74. diffusion_policy/config/rl_annotate.yaml +265 -0
  75. diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
  76. diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
  77. diffusion_policy/config/rl_annotate_v2.yaml +141 -0
  78. diffusion_policy/config/rl_annotate_v3.yaml +165 -0
  79. diffusion_policy/config/task/lift.yaml +89 -0
  80. diffusion_policy/config/task/square.yaml +92 -0
  81. diffusion_policy/config/task/tool_hang.yaml +93 -0
  82. diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
  83. diffusion_policy/config/task/umi.yaml +105 -0
  84. diffusion_policy/config/task/umi_bimanual.yaml +135 -0
  85. diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
  86. diffusion_policy/config/task/umi_image.yaml +36 -0
  87. diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
  88. diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
  89. diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
  90. diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
  91. diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
  92. diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
  93. diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
  94. diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
  95. diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
  96. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
  97. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
  98. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
  99. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
  100. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
  101. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
  102. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
  103. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
  104. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
  105. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
  106. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
  107. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
  108. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
  109. diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
  110. diffusion_policy/config/train_iql_umi.yaml +186 -0
  111. diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
  112. diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
  113. diffusion_policy/dataset/__init__.py +0 -0
  114. diffusion_policy/dataset/base_dataset.py +76 -0
  115. diffusion_policy/dataset/umi_dataset.py +652 -0
  116. diffusion_policy/env_runner/__init__.py +0 -0
  117. diffusion_policy/env_runner/base_image_runner.py +9 -0
  118. diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
  119. diffusion_policy/model/__init__.py +0 -0
  120. diffusion_policy/model/common/__init__.py +0 -0
  121. diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
  122. diffusion_policy/model/common/lr_scheduler.py +46 -0
  123. diffusion_policy/model/common/module_attr_mixin.py +14 -0
  124. diffusion_policy/model/common/normalizer.py +353 -0
  125. diffusion_policy/model/common/tensor_util.py +960 -0
  126. diffusion_policy/model/diffusion/__init__.py +0 -0
  127. diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
  128. diffusion_policy/model/diffusion/conv1d_components.py +46 -0
  129. diffusion_policy/model/diffusion/ema_model.py +88 -0
  130. diffusion_policy/model/diffusion/mask_generator.py +221 -0
  131. diffusion_policy/model/diffusion/positional_embedding.py +17 -0
  132. diffusion_policy/model/vision/__init__.py +0 -0
  133. diffusion_policy/model/vision/crop_randomizer.py +288 -0
  134. diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
  135. diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
  136. diffusion_policy/policy/__init__.py +0 -0
  137. diffusion_policy/policy/base_image_policy.py +27 -0
  138. diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
  139. diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
  140. diffusion_policy/policy/parl_action_optimizer.py +675 -0
  141. diffusion_policy/workspace/__init__.py +0 -0
  142. diffusion_policy/workspace/base_workspace.py +146 -0
  143. diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
  144. solarian_diffusion/__init__.py +32 -0
  145. solarian_diffusion/config.py +31 -0
  146. solarian_diffusion/policy_server.py +316 -0
  147. solarian_diffusion/rtc.py +272 -0
  148. solarian_diffusion/service.py +16 -0
  149. solarian_diffusion/trainer.py +213 -0
  150. solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
  151. solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
  152. solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
  153. solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
  154. solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
  155. solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
  156. solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
  157. umi/__init__.py +0 -0
  158. umi/common/__init__.py +0 -0
  159. umi/common/pose_util.py +115 -0
@@ -0,0 +1,254 @@
1
+ defaults:
2
+ - _self_
3
+ - task: umi_pick_lego
4
+
5
+ name: train_diffusion_unet_timm
6
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
7
+
8
+ task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+ batch_size: 128
12
+ # visual_encoder: "vit_base_patch16_clip_224.openai"
13
+ visual_encoder: "resnet34.a1_in1k"
14
+ model_type: "unet"
15
+ # this will be overridden at call time
16
+ num_gpus: 1
17
+
18
+
19
+ # Parsed from the dataset file name's hyphen-separated tokens.
20
+ # eg: pick_lego-black_mirror-real_200-sim_200-no_bbox.zarr.zip
21
+ # Resolver: train.py::parse_dataset_field. Last arg is the fallback default.
22
+ # mirror options: black_mirror, mirror, swap_mirror, no_mirror
23
+ mirror: ${parse_dataset_field:${task.dataset_path},mirror,no_mirror}
24
+ real_data_size: ${parse_dataset_field:${task.dataset_path},real_data_size,200}
25
+ sim_data_size: ${parse_dataset_field:${task.dataset_path},sim_data_size,200}
26
+ bbox: ${parse_dataset_field:${task.dataset_path},bbox,false}
27
+ data_task: ${parse_dataset_field:${task.dataset_path},data_task,pick_lego}
28
+
29
+ # Override dataset path for fine-tuning
30
+ # This will override the dataset_path in task/umi.yaml
31
+ # Uncomment and set to your new dataset path:
32
+ # task.dataset_path: "path/to/your/new/dataset.zarr.zip"
33
+ - "${eval:'\"vitb
34
+ n_action_steps: 8
35
+
36
+ policy:
37
+ _target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
38
+
39
+ shape_meta: ${shape_meta}
40
+
41
+ noise_scheduler:
42
+ _target_: diffusers.DDIMScheduler
43
+ num_train_timesteps: 50
44
+ beta_start: 0.0001
45
+ beta_end: 0.02
46
+ # beta_schedule is important
47
+ # this is the best we found
48
+ beta_schedule: squaredcos_cap_v2
49
+ clip_sample: True
50
+ set_alpha_to_one: True
51
+ steps_offset: 0
52
+ prediction_type: epsilon # or sample
53
+
54
+ obs_encoder:
55
+ _target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
56
+ shape_meta: ${shape_meta}
57
+
58
+ ##### from scratch #####
59
+ # model_name: 'vit_base_patch16_224'
60
+ # model_name: 'resnet34'
61
+ # model_name: 'vit_tiny_patch16_224'
62
+ # model_name: 'efficientnet_b0'
63
+ # model_name: 'efficientnet_b3'
64
+ # pretrained: False
65
+ # frozen: False
66
+
67
+ ##### from scratch #####
68
+ # model_name: 'resnet34.a1_in1k'
69
+ model_name: ${visual_encoder}
70
+ # model_name: 'vit_large_patch14_clip_224' # Changed to match checkpoint cup_wild_vit_l_1img.ckpt (ViT-L with CLIP weights)
71
+ # model_name: 'convnext_base.clip_laion2b_augreg_ft_in12k'
72
+ pretrained: True
73
+ frozen: False
74
+
75
+
76
+ global_pool: ''
77
+
78
+ # 'avg' or 'max' or 'soft_attention' or 'spatial_embedding' or 'transformer' or 'attention_pool_2d'
79
+ feature_aggregation: 'attention_pool_2d'
80
+ # feature_aggregation: null
81
+ position_encording: 'sinusoidal' # 'learnable' or 'sinusoidal'. it only works for transformer
82
+
83
+ # it only works for resnet. 32 (7x7) or 16 (14x14)
84
+ downsample_ratio: 32
85
+
86
+ transforms:
87
+ - type: RandomCrop
88
+ ratio: 0.95
89
+ # RandomRotation is not compatible with BF16
90
+ # - _target_: torchvision.transforms.RandomRotation
91
+ # degrees:
92
+ # - -5.0
93
+ # - 5.0
94
+ # expand: false
95
+ - _target_: torchvision.transforms.ColorJitter
96
+ brightness: 0.3
97
+ contrast: 0.4
98
+ saturation: 0.5
99
+ hue: 0.08
100
+ # - _target_: torchvision.transforms.RandomPerspective
101
+ # distortion_scale: 0.5
102
+ # p: 1.0
103
+ # - _target_: torchvision.transforms.ElasticTransform
104
+ # alpha: 50.0
105
+ # sigma: 5.0
106
+ # - _target_: torchvision.transforms.RandomPerspective
107
+ # distortion_scale: 0.5
108
+ # p: 1.0
109
+ # - _target_: torchvision.transforms.ElasticTransform
110
+ # alpha: 50.0
111
+ # sigma: 5.0
112
+ # - _target_: torchvision.transforms.RandomCrop
113
+ # size: 192
114
+
115
+
116
+ use_group_norm: True
117
+ share_rgb_model: False
118
+ imagenet_norm: True
119
+
120
+ num_inference_steps: 16
121
+ obs_as_global_cond: True
122
+ diffusion_step_embed_dim: 128
123
+ down_dims: [256,512,1024]
124
+ kernel_size: 5
125
+ n_groups: 8
126
+ cond_predict_scale: True
127
+ input_pertub: 0.1 # reference: https://github.com/forever208/DDPM-IP
128
+ # train multiple diffusion steps per obs encoder forward pass
129
+ # doing this too much will cause instability!
130
+ train_diffusion_n_samples: 1
131
+
132
+ ema:
133
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
134
+ update_after_step: 0
135
+ inv_gamma: 1.0
136
+ power: 0.75
137
+ min_value: 0.0
138
+ max_value: 0.9999
139
+
140
+ dataloader:
141
+ batch_size: ${batch_size}
142
+ num_workers: 16
143
+ shuffle: True
144
+ pin_memory: True
145
+ persistent_workers: True
146
+
147
+ val_dataloader:
148
+ batch_size: ${batch_size}
149
+ num_workers: 16
150
+ shuffle: False
151
+ pin_memory: True
152
+ persistent_workers: True
153
+
154
+ # dataloader:
155
+ # batch_size: 64
156
+ # num_workers: 16
157
+ # shuffle: True
158
+ # pin_memory: True
159
+ # persistent_workers: True
160
+
161
+ # val_dataloader:
162
+ # batch_size: 64
163
+ # num_workers: 16
164
+ # shuffle: False
165
+ # pin_memory: True
166
+ # persistent_workers: True
167
+
168
+ optimizer:
169
+ _target_: torch.optim.AdamW
170
+ lr: 3.0e-4
171
+ betas: [0.95, 0.999]
172
+ eps: 1.0e-8
173
+ weight_decay: 1.0e-6
174
+
175
+ training:
176
+ device: "cuda:0"
177
+ seed: 42
178
+ debug: False
179
+ resume: False
180
+ # optimization
181
+ lr_scheduler: cosine
182
+ lr_warmup_steps: 2000
183
+ num_epochs: 250
184
+ gradient_accumulate_every: 1
185
+ # EMA destroys performance when used with BatchNorm
186
+ # replace BatchNorm with GroupNorm.
187
+ use_ema: True
188
+ freeze_encoder: False
189
+ # training loop control
190
+ # in epochs
191
+ rollout_every: 10
192
+ checkpoint_every: 10
193
+ val_every: 1
194
+ sample_every: 5
195
+ # steps per epoch
196
+ max_train_steps: null
197
+ max_val_steps: null
198
+ # misc
199
+ tqdm_interval_sec: 1.0
200
+
201
+ logging:
202
+ project: umi
203
+ resume: False
204
+ mode: online
205
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
206
+ tags:
207
+ - "${name}"
208
+ - "${task_name}"
209
+ - "${exp_name}"
210
+ # visual encoder could be vit_base_patch16_clip_224.openai, resnet34, etc.
211
+ - "${eval:'\"vitb\" if \"${visual_encoder}\".startswith(\"vit_base\") else (\"vitl\" if \"${visual_encoder}\".startswith(\"vit_large\") else \"${visual_encoder}\")'}"
212
+ # batch size for each gpu
213
+ - "batch_size:${batch_size} * ${num_gpus}"
214
+ - "num_gpus:${num_gpus}"
215
+ # model type could be unet or transformer
216
+ - "${model_type}"
217
+ - "${task.dataset_path}"
218
+ # dataset file name (joined; substring-searchable on wandb)
219
+ - "${eval:'\"${task.dataset_path}\".split(\"/\")[-1].rsplit(\".\",1)[0]'}"
220
+ - "${mirror}"
221
+ - "real_size:${real_data_size}"
222
+ - "sim_size:${sim_data_size}"
223
+ - "${eval:'real' if int(${real_data_size}) > 0 else None}"
224
+ - "${eval:'sim' if int(${sim_data_size}) > 0 else None}"
225
+ - "${bbox}"
226
+ - "${data_task}"
227
+ real_data_size: ${parse_dataset_field:${task.dataset_path},real_data_size,200}
228
+ sim_data_size: ${parse_dataset_field:${task.dataset_path},sim_data_size,200}
229
+ bbox: ${parse_dataset_field:${task.dataset_path},bbox,false}
230
+ data_task: ${parse_dataset_field:${task.dataset_path},data_task,pick_lego}
231
+ id: null
232
+ group: ${task_name}_${model_type}
233
+
234
+ checkpoint:
235
+ topk:
236
+ monitor_key: train_loss
237
+ mode: min
238
+ k: 20
239
+ format_str: '{task_name}-{model_type}-{visual_encoder}-batch_{total_batch}-{dataset_file_name}-epoch_{epoch:04d}-train_loss_{train_loss:.3f}.ckpt'
240
+ save_last_ckpt: True
241
+ save_last_snapshot: False
242
+
243
+ multi_run:
244
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
245
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
246
+
247
+ hydra:
248
+ job:
249
+ override_dirname: ${name}
250
+ run:
251
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
252
+ sweep:
253
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
254
+ subdir: ${hydra.job.num}
@@ -0,0 +1,219 @@
1
+ defaults:
2
+ - task: umi
3
+ - _self_
4
+
5
+ name: train_diffusion_unet_timm
6
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
7
+
8
+ task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+
12
+ # Match checkpoint architecture (cup_wild_vit_l_1img.ckpt was trained with 1 image)
13
+ task:
14
+ img_obs_horizon: 1
15
+
16
+ # Override dataset path for fine-tuning
17
+ # This will override the dataset_path in task/umi.yaml
18
+ # Uncomment and set to your new dataset path:
19
+ # task.dataset_path: "path/to/your/new/dataset.zarr.zip"
20
+
21
+ n_action_steps: 8
22
+
23
+ policy:
24
+ _target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
25
+
26
+ shape_meta: ${shape_meta}
27
+
28
+ noise_scheduler:
29
+ _target_: diffusers.DDIMScheduler
30
+ num_train_timesteps: 50
31
+ beta_start: 0.0001
32
+ beta_end: 0.02
33
+ # beta_schedule is important
34
+ # this is the best we found
35
+ beta_schedule: squaredcos_cap_v2
36
+ clip_sample: True
37
+ set_alpha_to_one: True
38
+ steps_offset: 0
39
+ prediction_type: epsilon # or sample
40
+
41
+ obs_encoder:
42
+ _target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
43
+ shape_meta: ${shape_meta}
44
+
45
+ ##### from scratch #####
46
+ # model_name: 'vit_base_patch16_224'
47
+ # model_name: 'resnet34'
48
+ # model_name: 'vit_tiny_patch16_224'
49
+ # model_name: 'efficientnet_b0'
50
+ # model_name: 'efficientnet_b3'
51
+ # pretrained: False
52
+ # frozen: False
53
+
54
+ ##### from scratch #####
55
+ # model_name: 'resnet34.a1_in1k'
56
+ # model_name: 'vit_base_patch16_clip_224.openai'
57
+ model_name: 'vit_large_patch14_clip_224' # Changed to match checkpoint cup_wild_vit_l_1img.ckpt (ViT-L with CLIP weights)
58
+ # model_name: 'convnext_base.clip_laion2b_augreg_ft_in12k'
59
+ pretrained: True
60
+ frozen: False
61
+
62
+
63
+ global_pool: ''
64
+
65
+ # 'avg' or 'max' or 'soft_attention' or 'spatial_embedding' or 'transformer' or 'attention_pool_2d'
66
+ feature_aggregation: 'attention_pool_2d'
67
+ # feature_aggregation: null
68
+ position_encording: 'sinusoidal' # 'learnable' or 'sinusoidal'. it only works for transformer
69
+
70
+ # it only works for resnet. 32 (7x7) or 16 (14x14)
71
+ downsample_ratio: 32
72
+
73
+ transforms:
74
+ - type: RandomCrop
75
+ ratio: 0.95
76
+ # RandomRotation is not compatible with BF16
77
+ # - _target_: torchvision.transforms.RandomRotation
78
+ # degrees:
79
+ # - -5.0
80
+ # - 5.0
81
+ # expand: false
82
+ - _target_: torchvision.transforms.ColorJitter
83
+ brightness: 0.2 # Reduced augmentation for fine-tuning (was 0.3)
84
+ contrast: 0.2 # Reduced augmentation for fine-tuning (was 0.4)
85
+ saturation: 0.2 # Reduced augmentation for fine-tuning (was 0.5)
86
+ hue: 0.05 # Reduced augmentation for fine-tuning (was 0.08)
87
+ # - _target_: torchvision.transforms.RandomPerspective
88
+ # distortion_scale: 0.5
89
+ # p: 1.0
90
+ # - _target_: torchvision.transforms.ElasticTransform
91
+ # alpha: 50.0
92
+ # sigma: 5.0
93
+ # - _target_: torchvision.transforms.RandomPerspective
94
+ # distortion_scale: 0.5
95
+ # p: 1.0
96
+ # - _target_: torchvision.transforms.ElasticTransform
97
+ # alpha: 50.0
98
+ # sigma: 5.0
99
+ # - _target_: torchvision.transforms.RandomCrop
100
+ # size: 192
101
+
102
+
103
+ use_group_norm: True
104
+ share_rgb_model: False
105
+ imagenet_norm: True
106
+
107
+ num_inference_steps: 16
108
+ obs_as_global_cond: True
109
+ diffusion_step_embed_dim: 128
110
+ down_dims: [256,512,1024]
111
+ kernel_size: 5
112
+ n_groups: 8
113
+ cond_predict_scale: True
114
+ input_pertub: 0.1 # reference: https://github.com/forever208/DDPM-IP
115
+ # train multiple diffusion steps per obs encoder forward pass
116
+ # doing this too much will cause instability!
117
+ train_diffusion_n_samples: 1
118
+
119
+ ema:
120
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
121
+ update_after_step: 0
122
+ inv_gamma: 1.0
123
+ power: 0.75
124
+ min_value: 0.0
125
+ max_value: 0.9999
126
+
127
+ dataloader:
128
+ batch_size: 8 # Reduced for small dataset (50 examples)
129
+ num_workers: 4
130
+ shuffle: True
131
+ pin_memory: True
132
+ persistent_workers: True
133
+
134
+ val_dataloader:
135
+ batch_size: 8 # Reduced for small dataset
136
+ num_workers: 4
137
+ shuffle: False
138
+ pin_memory: True
139
+ persistent_workers: True
140
+
141
+ # dataloader:
142
+ # batch_size: 64
143
+ # num_workers: 16
144
+ # shuffle: True
145
+ # pin_memory: True
146
+ # persistent_workers: True
147
+
148
+ # val_dataloader:
149
+ # batch_size: 64
150
+ # num_workers: 16
151
+ # shuffle: False
152
+ # pin_memory: True
153
+ # persistent_workers: True
154
+
155
+ optimizer:
156
+ _target_: torch.optim.AdamW
157
+ lr: 3.0e-5 # Reduced learning rate for fine-tuning (10x smaller)
158
+ betas: [0.95, 0.999]
159
+ eps: 1.0e-8
160
+ weight_decay: 1.0e-6
161
+
162
+ training:
163
+ device: "cuda:0"
164
+ seed: 42
165
+ debug: False
166
+ resume: True # Enable resume to load checkpoint
167
+ checkpoint_path: data/models/cup_wild_vit_l_1img.ckpt # Path to your pretrained model
168
+ # Example: checkpoint_path: "data/outputs/2024.01.15/12.30.45_train_diffusion_unet_timm_umi/checkpoints/latest.ckpt"
169
+ # optimization
170
+ lr_scheduler: cosine
171
+ lr_warmup_steps: 50 # Reduced warmup for small dataset (was 2000)
172
+ num_epochs: 50 # Fewer epochs for fine-tuning (was 120)
173
+ gradient_accumulate_every: 1
174
+ # EMA destroys performance when used with BatchNorm
175
+ # replace BatchNorm with GroupNorm.
176
+ use_ema: True
177
+ freeze_encoder: True # Freeze encoder to prevent overfitting on small dataset
178
+ # training loop control
179
+ # in epochs
180
+ rollout_every: 10
181
+ checkpoint_every: 5 # More frequent checkpoints for fine-tuning
182
+ val_every: 1 # Validate every epoch to catch overfitting early
183
+ sample_every: 5
184
+ # steps per epoch
185
+ max_train_steps: null
186
+ max_val_steps: null
187
+ # misc
188
+ tqdm_interval_sec: 1.0
189
+
190
+ logging:
191
+ project: umi
192
+ resume: False
193
+ mode: online
194
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
195
+ tags: ["${name}", "${task_name}", "${exp_name}"]
196
+ id: null
197
+ group: null
198
+
199
+ checkpoint:
200
+ topk:
201
+ monitor_key: train_loss
202
+ mode: min
203
+ k: 20
204
+ format_str: 'epoch_{epoch:04d}-train_loss_{train_loss:.3f}.ckpt'
205
+ save_last_ckpt: True
206
+ save_last_snapshot: False
207
+
208
+ multi_run:
209
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
210
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
211
+
212
+ hydra:
213
+ job:
214
+ override_dirname: ${name}
215
+ run:
216
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
217
+ sweep:
218
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
219
+ subdir: ${hydra.job.num}
@@ -0,0 +1,203 @@
1
+ defaults:
2
+ - _self_
3
+ - task: umi_bimanual
4
+
5
+ name: train_diffusion_unet_timm
6
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
7
+
8
+ task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+
12
+ n_action_steps: 8
13
+
14
+ policy:
15
+ _target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
16
+
17
+ shape_meta: ${shape_meta}
18
+
19
+ noise_scheduler:
20
+ _target_: diffusers.DDIMScheduler
21
+ num_train_timesteps: 50
22
+ beta_start: 0.0001
23
+ beta_end: 0.02
24
+ # beta_schedule is important
25
+ # this is the best we found
26
+ beta_schedule: squaredcos_cap_v2
27
+ clip_sample: True
28
+ set_alpha_to_one: True
29
+ steps_offset: 0
30
+ prediction_type: epsilon # or sample
31
+
32
+ obs_encoder:
33
+ _target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
34
+ shape_meta: ${shape_meta}
35
+
36
+ ##### from scratch #####
37
+ # model_name: 'vit_base_patch16_224'
38
+ # model_name: 'resnet34'
39
+ # model_name: 'vit_tiny_patch16_224'
40
+ # model_name: 'efficientnet_b0'
41
+ # model_name: 'efficientnet_b3'
42
+ # pretrained: False
43
+ # frozen: False
44
+
45
+ ##### from scratch #####
46
+ # model_name: 'resnet34.a1_in1k'
47
+ model_name: 'vit_base_patch16_clip_224.openai'
48
+ # model_name: 'convnext_base.clip_laion2b_augreg_ft_in12k'
49
+ pretrained: True
50
+ frozen: False
51
+
52
+
53
+ global_pool: ''
54
+
55
+ # 'avg' or 'max' or 'soft_attention' or 'spatial_embedding' or 'transformer' or 'attention_pool_2d'
56
+ feature_aggregation: 'attention_pool_2d'
57
+ # feature_aggregation: null
58
+ position_encording: 'sinusoidal' # 'learnable' or 'sinusoidal'. it only works for transformer
59
+
60
+ # it only works for resnet. 32 (7x7) or 16 (14x14)
61
+ downsample_ratio: 32
62
+
63
+ transforms:
64
+ - type: RandomCrop
65
+ ratio: 0.95
66
+ - _target_: torchvision.transforms.RandomRotation
67
+ degrees:
68
+ - -5.0
69
+ - 5.0
70
+ expand: false
71
+ - _target_: torchvision.transforms.ColorJitter
72
+ brightness: 0.3
73
+ contrast: 0.4
74
+ saturation: 0.5
75
+ hue: 0.08
76
+ # - _target_: torchvision.transforms.RandomPerspective
77
+ # distortion_scale: 0.5
78
+ # p: 1.0
79
+ # - _target_: torchvision.transforms.ElasticTransform
80
+ # alpha: 50.0
81
+ # sigma: 5.0
82
+ # - _target_: torchvision.transforms.RandomPerspective
83
+ # distortion_scale: 0.5
84
+ # p: 1.0
85
+ # - _target_: torchvision.transforms.ElasticTransform
86
+ # alpha: 50.0
87
+ # sigma: 5.0
88
+ # - _target_: torchvision.transforms.RandomCrop
89
+ # size: 192
90
+
91
+
92
+ use_group_norm: True
93
+ share_rgb_model: False
94
+ imagenet_norm: True
95
+
96
+ num_inference_steps: 16
97
+ obs_as_global_cond: True
98
+ diffusion_step_embed_dim: 128
99
+ down_dims: [256,512,1024]
100
+ kernel_size: 5
101
+ n_groups: 8
102
+ cond_predict_scale: True
103
+ input_pertub: 0.1 # reference: https://github.com/forever208/DDPM-IP
104
+
105
+ ema:
106
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
107
+ update_after_step: 0
108
+ inv_gamma: 1.0
109
+ power: 0.75
110
+ min_value: 0.0
111
+ max_value: 0.9999
112
+
113
+ dataloader:
114
+ batch_size: 32
115
+ num_workers: 8
116
+ shuffle: True
117
+ pin_memory: True
118
+ persistent_workers: True
119
+
120
+ val_dataloader:
121
+ batch_size: 32
122
+ num_workers: 8
123
+ shuffle: False
124
+ pin_memory: True
125
+ persistent_workers: True
126
+
127
+ # dataloader:
128
+ # batch_size: 64
129
+ # num_workers: 16
130
+ # shuffle: True
131
+ # pin_memory: True
132
+ # persistent_workers: True
133
+
134
+ # val_dataloader:
135
+ # batch_size: 64
136
+ # num_workers: 16
137
+ # shuffle: False
138
+ # pin_memory: True
139
+ # persistent_workers: True
140
+
141
+ optimizer:
142
+ _target_: torch.optim.AdamW
143
+ lr: 3.0e-4
144
+ betas: [0.95, 0.999]
145
+ eps: 1.0e-8
146
+ weight_decay: 1.0e-6
147
+
148
+ training:
149
+ device: "cuda:0"
150
+ seed: 42
151
+ debug: False
152
+ resume: False
153
+ # optimization
154
+ lr_scheduler: cosine
155
+ lr_warmup_steps: 2000
156
+ num_epochs: 100
157
+ gradient_accumulate_every: 1
158
+ # EMA destroys performance when used with BatchNorm
159
+ # replace BatchNorm with GroupNorm.
160
+ use_ema: True
161
+ freeze_encoder: False
162
+ # training loop control
163
+ # in epochs
164
+ rollout_every: 10
165
+ checkpoint_every: 10
166
+ val_every: 5
167
+ sample_every: 5
168
+ # steps per epoch
169
+ max_train_steps: null
170
+ max_val_steps: null
171
+ # misc
172
+ tqdm_interval_sec: 1.0
173
+
174
+ logging:
175
+ project: umi
176
+ resume: False
177
+ mode: online
178
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
179
+ tags: ["${name}", "${task_name}", "${exp_name}"]
180
+ id: null
181
+ group: null
182
+
183
+ checkpoint:
184
+ topk:
185
+ monitor_key: train_loss
186
+ mode: min
187
+ k: 20
188
+ format_str: 'epoch_{epoch:04d}-train_loss_{train_loss:.3f}.ckpt'
189
+ save_last_ckpt: True
190
+ save_last_snapshot: False
191
+
192
+ multi_run:
193
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
194
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
195
+
196
+ hydra:
197
+ job:
198
+ override_dirname: ${name}
199
+ run:
200
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
201
+ sweep:
202
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
203
+ subdir: ${hydra.job.num}