solarian-diffusion 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. diffusion_policy/__init__.py +0 -0
  2. diffusion_policy/codecs/__init__.py +0 -0
  3. diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
  4. diffusion_policy/common/__init__.py +0 -0
  5. diffusion_policy/common/checkpoint_util.py +64 -0
  6. diffusion_policy/common/json_logger.py +117 -0
  7. diffusion_policy/common/normalize_util.py +251 -0
  8. diffusion_policy/common/pose_repr_util.py +112 -0
  9. diffusion_policy/common/pytorch_util.py +82 -0
  10. diffusion_policy/common/q_critic_util.py +135 -0
  11. diffusion_policy/common/replay_buffer.py +672 -0
  12. diffusion_policy/common/sampler.py +204 -0
  13. diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
  14. diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
  15. diffusion_policy/config/legacy/task/can_image.yaml +64 -0
  16. diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
  17. diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
  18. diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
  19. diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
  20. diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
  21. diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
  22. diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
  23. diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
  24. diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
  25. diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
  26. diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
  27. diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
  28. diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
  29. diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
  30. diffusion_policy/config/legacy/task/square_image.yaml +64 -0
  31. diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
  32. diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
  33. diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
  34. diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
  35. diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
  36. diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
  37. diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
  38. diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
  39. diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
  40. diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
  41. diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
  42. diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
  43. diffusion_policy/config/legacy/task/umi.yaml +45 -0
  44. diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
  45. diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
  46. diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
  47. diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
  48. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
  49. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
  50. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
  51. diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
  52. diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
  53. diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
  54. diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
  55. diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
  56. diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
  57. diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
  58. diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
  59. diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
  60. diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
  61. diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
  62. diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
  63. diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
  64. diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
  65. diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
  66. diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
  67. diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
  68. diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
  69. diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
  70. diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
  71. diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
  72. diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
  73. diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
  74. diffusion_policy/config/rl_annotate.yaml +265 -0
  75. diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
  76. diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
  77. diffusion_policy/config/rl_annotate_v2.yaml +141 -0
  78. diffusion_policy/config/rl_annotate_v3.yaml +165 -0
  79. diffusion_policy/config/task/lift.yaml +89 -0
  80. diffusion_policy/config/task/square.yaml +92 -0
  81. diffusion_policy/config/task/tool_hang.yaml +93 -0
  82. diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
  83. diffusion_policy/config/task/umi.yaml +105 -0
  84. diffusion_policy/config/task/umi_bimanual.yaml +135 -0
  85. diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
  86. diffusion_policy/config/task/umi_image.yaml +36 -0
  87. diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
  88. diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
  89. diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
  90. diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
  91. diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
  92. diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
  93. diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
  94. diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
  95. diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
  96. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
  97. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
  98. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
  99. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
  100. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
  101. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
  102. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
  103. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
  104. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
  105. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
  106. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
  107. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
  108. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
  109. diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
  110. diffusion_policy/config/train_iql_umi.yaml +186 -0
  111. diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
  112. diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
  113. diffusion_policy/dataset/__init__.py +0 -0
  114. diffusion_policy/dataset/base_dataset.py +76 -0
  115. diffusion_policy/dataset/umi_dataset.py +652 -0
  116. diffusion_policy/env_runner/__init__.py +0 -0
  117. diffusion_policy/env_runner/base_image_runner.py +9 -0
  118. diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
  119. diffusion_policy/model/__init__.py +0 -0
  120. diffusion_policy/model/common/__init__.py +0 -0
  121. diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
  122. diffusion_policy/model/common/lr_scheduler.py +46 -0
  123. diffusion_policy/model/common/module_attr_mixin.py +14 -0
  124. diffusion_policy/model/common/normalizer.py +353 -0
  125. diffusion_policy/model/common/tensor_util.py +960 -0
  126. diffusion_policy/model/diffusion/__init__.py +0 -0
  127. diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
  128. diffusion_policy/model/diffusion/conv1d_components.py +46 -0
  129. diffusion_policy/model/diffusion/ema_model.py +88 -0
  130. diffusion_policy/model/diffusion/mask_generator.py +221 -0
  131. diffusion_policy/model/diffusion/positional_embedding.py +17 -0
  132. diffusion_policy/model/vision/__init__.py +0 -0
  133. diffusion_policy/model/vision/crop_randomizer.py +288 -0
  134. diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
  135. diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
  136. diffusion_policy/policy/__init__.py +0 -0
  137. diffusion_policy/policy/base_image_policy.py +27 -0
  138. diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
  139. diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
  140. diffusion_policy/policy/parl_action_optimizer.py +675 -0
  141. diffusion_policy/workspace/__init__.py +0 -0
  142. diffusion_policy/workspace/base_workspace.py +146 -0
  143. diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
  144. solarian_diffusion/__init__.py +32 -0
  145. solarian_diffusion/config.py +31 -0
  146. solarian_diffusion/policy_server.py +316 -0
  147. solarian_diffusion/rtc.py +272 -0
  148. solarian_diffusion/service.py +16 -0
  149. solarian_diffusion/trainer.py +213 -0
  150. solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
  151. solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
  152. solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
  153. solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
  154. solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
  155. solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
  156. solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
  157. umi/__init__.py +0 -0
  158. umi/common/__init__.py +0 -0
  159. umi/common/pose_util.py +115 -0
@@ -0,0 +1,260 @@
1
+ defaults:
2
+ - _self_
3
+ - task: umi_pick_single_lego_175
4
+
5
+ name: train_diffusion_unet_timm
6
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
7
+
8
+ # task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+ batch_size: 128
12
+ # visual_encoder: "vit_base_patch16_clip_224.openai"
13
+ # visual_encoder_tag: "vitb"
14
+ visual_encoder: "vit_large_patch14_clip_224"
15
+ visual_encoder_tag: "vitl"
16
+ # visual_encoder: "resnet34.a1_in1k"
17
+ # visual_encoder_tag: "resnet34"
18
+ model_type: "unet"
19
+ # this will be overridden at call time
20
+ num_gpus: 1
21
+
22
+
23
+ # Parsed from the dataset file name's hyphen-separated tokens.
24
+ # eg: pick_lego-black_mirror-real_200-sim_200-no_bbox.zarr.zip
25
+ # Resolver: train.py::parse_dataset_field. Last arg is the fallback default.
26
+ # mirror options: black_mirror, mirror, swap_mirror, no_mirror
27
+ mirror: ${parse_dataset_field:${task.dataset_path},mirror,no_mirror}
28
+ # real_data_size / sim_data_size are null when the filename has no real_<N>/sim_<N>
29
+ # token; the wandb tags below skip themselves in that case.
30
+ real_data_size: ${parse_dataset_field:${task.dataset_path},real_data_size,null}
31
+ sim_data_size: ${parse_dataset_field:${task.dataset_path},sim_data_size,null}
32
+ bbox: ${parse_dataset_field:${task.dataset_path},bbox,no_bbox}
33
+ task_name: ${parse_dataset_field:${task.dataset_path},data_task,pick_lego}
34
+
35
+ # Override dataset path for fine-tuning
36
+ # This will override the dataset_path in task/umi.yaml
37
+ # Uncomment and set to your new dataset path:
38
+ # task.dataset_path: "path/to/your/new/dataset.zarr.zip"
39
+
40
+ n_action_steps: 8
41
+
42
+ policy:
43
+ _target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
44
+
45
+ shape_meta: ${shape_meta}
46
+
47
+ noise_scheduler:
48
+ _target_: diffusers.DDIMScheduler
49
+ num_train_timesteps: 50
50
+ beta_start: 0.0001
51
+ beta_end: 0.02
52
+ # beta_schedule is important
53
+ # this is the best we found
54
+ beta_schedule: squaredcos_cap_v2
55
+ clip_sample: True
56
+ set_alpha_to_one: True
57
+ steps_offset: 0
58
+ prediction_type: epsilon # or sample
59
+
60
+ obs_encoder:
61
+ _target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
62
+ shape_meta: ${shape_meta}
63
+
64
+ ##### from scratch #####
65
+ # model_name: 'vit_base_patch16_224'
66
+ # model_name: 'resnet34'
67
+ # model_name: 'vit_tiny_patch16_224'
68
+ # model_name: 'efficientnet_b0'
69
+ # model_name: 'efficientnet_b3'
70
+ # pretrained: False
71
+ # frozen: False
72
+
73
+ ##### from scratch #####
74
+ # model_name: 'resnet34.a1_in1k'
75
+ model_name: ${visual_encoder}
76
+ # model_name: 'vit_large_patch14_clip_224' # Changed to match checkpoint cup_wild_vit_l_1img.ckpt (ViT-L with CLIP weights)
77
+ # model_name: 'convnext_base.clip_laion2b_augreg_ft_in12k'
78
+ pretrained: True
79
+ frozen: False
80
+
81
+
82
+ global_pool: ''
83
+
84
+ # 'avg' or 'max' or 'soft_attention' or 'spatial_embedding' or 'transformer' or 'attention_pool_2d'
85
+ feature_aggregation: 'attention_pool_2d'
86
+ # feature_aggregation: null
87
+ position_encording: 'sinusoidal' # 'learnable' or 'sinusoidal'. it only works for transformer
88
+
89
+ # it only works for resnet. 32 (7x7) or 16 (14x14)
90
+ downsample_ratio: 32
91
+
92
+ transforms:
93
+ - type: RandomCrop
94
+ ratio: 0.95
95
+ # RandomRotation is not compatible with BF16
96
+ # - _target_: torchvision.transforms.RandomRotation
97
+ # degrees:
98
+ # - -5.0
99
+ # - 5.0
100
+ # expand: false
101
+ - _target_: torchvision.transforms.ColorJitter
102
+ brightness: 0.3
103
+ contrast: 0.4
104
+ saturation: 0.5
105
+ hue: 0.08
106
+ # - _target_: torchvision.transforms.RandomPerspective
107
+ # distortion_scale: 0.5
108
+ # p: 1.0
109
+ # - _target_: torchvision.transforms.ElasticTransform
110
+ # alpha: 50.0
111
+ # sigma: 5.0
112
+ # - _target_: torchvision.transforms.RandomPerspective
113
+ # distortion_scale: 0.5
114
+ # p: 1.0
115
+ # - _target_: torchvision.transforms.ElasticTransform
116
+ # alpha: 50.0
117
+ # sigma: 5.0
118
+ # - _target_: torchvision.transforms.RandomCrop
119
+ # size: 192
120
+
121
+
122
+ use_group_norm: True
123
+ share_rgb_model: False
124
+ imagenet_norm: True
125
+
126
+ num_inference_steps: 16
127
+ obs_as_global_cond: True
128
+ diffusion_step_embed_dim: 128
129
+ down_dims: [256,512,1024]
130
+ kernel_size: 5
131
+ n_groups: 8
132
+ cond_predict_scale: True
133
+ input_pertub: 0.1 # reference: https://github.com/forever208/DDPM-IP
134
+ # train multiple diffusion steps per obs encoder forward pass
135
+ # doing this too much will cause instability!
136
+ train_diffusion_n_samples: 1
137
+
138
+ ema:
139
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
140
+ update_after_step: 0
141
+ inv_gamma: 1.0
142
+ power: 0.75
143
+ min_value: 0.0
144
+ max_value: 0.9999
145
+
146
+ dataloader:
147
+ batch_size: ${batch_size}
148
+ num_workers: 8
149
+ shuffle: True
150
+ pin_memory: True
151
+ persistent_workers: True
152
+ prefetch_factor: 4
153
+
154
+ val_dataloader:
155
+ batch_size: ${batch_size}
156
+ num_workers: 8
157
+ shuffle: False
158
+ pin_memory: True
159
+ persistent_workers: True
160
+ prefetch_factor: 4
161
+
162
+ # dataloader:
163
+ # batch_size: 64
164
+ # num_workers: 16
165
+ # shuffle: True
166
+ # pin_memory: True
167
+ # persistent_workers: True
168
+
169
+ # val_dataloader:
170
+ # batch_size: 64
171
+ # num_workers: 16
172
+ # shuffle: False
173
+ # pin_memory: True
174
+ # persistent_workers: True
175
+
176
+ optimizer:
177
+ _target_: torch.optim.AdamW
178
+ lr: 3.0e-4
179
+ betas: [0.95, 0.999]
180
+ eps: 1.0e-8
181
+ weight_decay: 1.0e-6
182
+
183
+ training:
184
+ device: "cuda:0"
185
+ seed: 42
186
+ debug: False
187
+ resume: False
188
+ # optimization
189
+ lr_scheduler: cosine
190
+ lr_warmup_steps: 500
191
+ num_epochs: 250
192
+ gradient_accumulate_every: 1
193
+ # EMA destroys performance when used with BatchNorm
194
+ # replace BatchNorm with GroupNorm.
195
+ use_ema: True
196
+ freeze_encoder: False
197
+ # training loop control
198
+ # in epochs
199
+ rollout_every: 10
200
+ checkpoint_every: 10
201
+ val_every: 1
202
+ sample_every: 5
203
+ # steps per epoch
204
+ max_train_steps: null
205
+ max_val_steps: null
206
+ # misc
207
+ tqdm_interval_sec: 1.0
208
+
209
+ logging:
210
+ project: umi
211
+ resume: False
212
+ mode: online
213
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
214
+ tags:
215
+ - "${name}"
216
+ - "${task_name}"
217
+ - "${exp_name}"
218
+ # visual encoder could be vit_base_patch16_clip_224.openai, resnet34, etc.
219
+ - "${visual_encoder_tag}"
220
+ # batch size for each gpu
221
+ - "batch_size:${eval:'${batch_size} * ${num_gpus}'}"
222
+ - "num_gpus:${num_gpus}"
223
+ # model type could be unet or transformer
224
+ - "${model_type}"
225
+ # dataset file name (joined; substring-searchable on wandb)
226
+ # Note: full ${task.dataset_path} dropped — wandb tags max 64 chars.
227
+ - "${eval:'\"${task.dataset_path}\".split(\"/\")[-1].rsplit(\".\",1)[0]'}"
228
+ - "${mirror}"
229
+ # Only emit real_/sim_ tags when the dataset filename actually carries
230
+ # a real_<N> / sim_<N> token (otherwise the resolver returns None and we
231
+ # render an empty string here, which is filtered out before wandb init).
232
+ - "${eval:'\"real_size:%d\" % ${real_data_size} if ${real_data_size} is not None else \"\"'}"
233
+ - "${eval:'\"sim_size:%d\" % ${sim_data_size} if ${sim_data_size} is not None else \"\"'}"
234
+ - "${eval:'(\"real\" if ${real_data_size} > 0 else \"no_real\") if ${real_data_size} is not None else \"\"'}"
235
+ - "${eval:'(\"sim\" if ${sim_data_size} > 0 else \"no_sim\") if ${sim_data_size} is not None else \"\"'}"
236
+ - "${bbox}"
237
+ id: null
238
+ group: ${task_name}_${model_type}
239
+
240
+ checkpoint:
241
+ topk:
242
+ monitor_key: train_loss
243
+ mode: min
244
+ k: 20
245
+ format_str: '{task_name}-{model_type}-{visual_encoder}-batch_{total_batch}-{dataset_file_name}-epoch_{epoch:04d}-train_loss_{train_loss:.3f}.ckpt'
246
+ save_last_ckpt: True
247
+ save_last_snapshot: False
248
+
249
+ multi_run:
250
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
251
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
252
+
253
+ hydra:
254
+ job:
255
+ override_dirname: ${name}
256
+ run:
257
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
258
+ sweep:
259
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
260
+ subdir: ${hydra.job.num}
@@ -0,0 +1,260 @@
1
+ defaults:
2
+ - _self_
3
+ - task: umi_pick_single_lego_175
4
+
5
+ name: train_diffusion_unet_timm
6
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
7
+
8
+ # task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+ batch_size: 128
12
+ # visual_encoder: "vit_base_patch16_clip_224.openai"
13
+ # visual_encoder_tag: "vitb"
14
+ visual_encoder: "vit_large_patch14_clip_224"
15
+ visual_encoder_tag: "vitl"
16
+ # visual_encoder: "resnet34.a1_in1k"
17
+ # visual_encoder_tag: "resnet34"
18
+ model_type: "unet"
19
+ # this will be overridden at call time
20
+ num_gpus: 1
21
+
22
+
23
+ # Parsed from the dataset file name's hyphen-separated tokens.
24
+ # eg: pick_lego-black_mirror-real_200-sim_200-no_bbox.zarr.zip
25
+ # Resolver: train.py::parse_dataset_field. Last arg is the fallback default.
26
+ # mirror options: black_mirror, mirror, swap_mirror, no_mirror
27
+ mirror: ${parse_dataset_field:${task.dataset_path},mirror,no_mirror}
28
+ # real_data_size / sim_data_size are null when the filename has no real_<N>/sim_<N>
29
+ # token; the wandb tags below skip themselves in that case.
30
+ real_data_size: ${parse_dataset_field:${task.dataset_path},real_data_size,null}
31
+ sim_data_size: ${parse_dataset_field:${task.dataset_path},sim_data_size,null}
32
+ bbox: ${parse_dataset_field:${task.dataset_path},bbox,no_bbox}
33
+ task_name: ${parse_dataset_field:${task.dataset_path},data_task,pick_lego}
34
+
35
+ # Override dataset path for fine-tuning
36
+ # This will override the dataset_path in task/umi.yaml
37
+ # Uncomment and set to your new dataset path:
38
+ # task.dataset_path: "path/to/your/new/dataset.zarr.zip"
39
+
40
+ n_action_steps: 8
41
+
42
+ policy:
43
+ _target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
44
+
45
+ shape_meta: ${shape_meta}
46
+
47
+ noise_scheduler:
48
+ _target_: diffusers.DDIMScheduler
49
+ num_train_timesteps: 50
50
+ beta_start: 0.0001
51
+ beta_end: 0.02
52
+ # beta_schedule is important
53
+ # this is the best we found
54
+ beta_schedule: squaredcos_cap_v2
55
+ clip_sample: True
56
+ set_alpha_to_one: True
57
+ steps_offset: 0
58
+ prediction_type: epsilon # or sample
59
+
60
+ obs_encoder:
61
+ _target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
62
+ shape_meta: ${shape_meta}
63
+
64
+ ##### from scratch #####
65
+ # model_name: 'vit_base_patch16_224'
66
+ # model_name: 'resnet34'
67
+ # model_name: 'vit_tiny_patch16_224'
68
+ # model_name: 'efficientnet_b0'
69
+ # model_name: 'efficientnet_b3'
70
+ # pretrained: False
71
+ # frozen: False
72
+
73
+ ##### from scratch #####
74
+ # model_name: 'resnet34.a1_in1k'
75
+ model_name: ${visual_encoder}
76
+ # model_name: 'vit_large_patch14_clip_224' # Changed to match checkpoint cup_wild_vit_l_1img.ckpt (ViT-L with CLIP weights)
77
+ # model_name: 'convnext_base.clip_laion2b_augreg_ft_in12k'
78
+ pretrained: True
79
+ frozen: False
80
+
81
+
82
+ global_pool: ''
83
+
84
+ # 'avg' or 'max' or 'soft_attention' or 'spatial_embedding' or 'transformer' or 'attention_pool_2d'
85
+ feature_aggregation: 'attention_pool_2d'
86
+ # feature_aggregation: null
87
+ position_encording: 'sinusoidal' # 'learnable' or 'sinusoidal'. it only works for transformer
88
+
89
+ # it only works for resnet. 32 (7x7) or 16 (14x14)
90
+ downsample_ratio: 32
91
+
92
+ transforms:
93
+ - type: RandomCrop
94
+ ratio: 0.95
95
+ # RandomRotation is not compatible with BF16
96
+ # - _target_: torchvision.transforms.RandomRotation
97
+ # degrees:
98
+ # - -5.0
99
+ # - 5.0
100
+ # expand: false
101
+ - _target_: torchvision.transforms.ColorJitter
102
+ brightness: 0.3
103
+ contrast: 0.4
104
+ saturation: 0.5
105
+ hue: 0.08
106
+ # - _target_: torchvision.transforms.RandomPerspective
107
+ # distortion_scale: 0.5
108
+ # p: 1.0
109
+ # - _target_: torchvision.transforms.ElasticTransform
110
+ # alpha: 50.0
111
+ # sigma: 5.0
112
+ # - _target_: torchvision.transforms.RandomPerspective
113
+ # distortion_scale: 0.5
114
+ # p: 1.0
115
+ # - _target_: torchvision.transforms.ElasticTransform
116
+ # alpha: 50.0
117
+ # sigma: 5.0
118
+ # - _target_: torchvision.transforms.RandomCrop
119
+ # size: 192
120
+
121
+
122
+ use_group_norm: True
123
+ share_rgb_model: False
124
+ imagenet_norm: True
125
+
126
+ num_inference_steps: 16
127
+ obs_as_global_cond: True
128
+ diffusion_step_embed_dim: 128
129
+ down_dims: [256,512,1024]
130
+ kernel_size: 5
131
+ n_groups: 8
132
+ cond_predict_scale: True
133
+ input_pertub: 0.1 # reference: https://github.com/forever208/DDPM-IP
134
+ # train multiple diffusion steps per obs encoder forward pass
135
+ # doing this too much will cause instability!
136
+ train_diffusion_n_samples: 1
137
+
138
+ ema:
139
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
140
+ update_after_step: 0
141
+ inv_gamma: 1.0
142
+ power: 0.75
143
+ min_value: 0.0
144
+ max_value: 0.9999
145
+
146
+ dataloader:
147
+ batch_size: ${batch_size}
148
+ num_workers: 8
149
+ shuffle: True
150
+ pin_memory: True
151
+ persistent_workers: True
152
+ prefetch_factor: 4
153
+
154
+ val_dataloader:
155
+ batch_size: ${batch_size}
156
+ num_workers: 8
157
+ shuffle: False
158
+ pin_memory: True
159
+ persistent_workers: True
160
+ prefetch_factor: 4
161
+
162
+ # dataloader:
163
+ # batch_size: 64
164
+ # num_workers: 16
165
+ # shuffle: True
166
+ # pin_memory: True
167
+ # persistent_workers: True
168
+
169
+ # val_dataloader:
170
+ # batch_size: 64
171
+ # num_workers: 16
172
+ # shuffle: False
173
+ # pin_memory: True
174
+ # persistent_workers: True
175
+
176
+ optimizer:
177
+ _target_: torch.optim.AdamW
178
+ lr: 3.0e-4
179
+ betas: [0.95, 0.999]
180
+ eps: 1.0e-8
181
+ weight_decay: 1.0e-6
182
+
183
+ training:
184
+ device: "cuda:0"
185
+ seed: 42
186
+ debug: False
187
+ resume: False
188
+ # optimization
189
+ lr_scheduler: cosine
190
+ lr_warmup_steps: 500
191
+ num_epochs: 250
192
+ gradient_accumulate_every: 1
193
+ # EMA destroys performance when used with BatchNorm
194
+ # replace BatchNorm with GroupNorm.
195
+ use_ema: True
196
+ freeze_encoder: False
197
+ # training loop control
198
+ # in epochs
199
+ rollout_every: 10
200
+ checkpoint_every: 10
201
+ val_every: 1
202
+ sample_every: 5
203
+ # steps per epoch
204
+ max_train_steps: null
205
+ max_val_steps: null
206
+ # misc
207
+ tqdm_interval_sec: 1.0
208
+
209
+ logging:
210
+ project: umi
211
+ resume: False
212
+ mode: online
213
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
214
+ tags:
215
+ - "${name}"
216
+ - "${task_name}"
217
+ - "${exp_name}"
218
+ # visual encoder could be vit_base_patch16_clip_224.openai, resnet34, etc.
219
+ - "${visual_encoder_tag}"
220
+ # batch size for each gpu
221
+ - "batch_size:${eval:'${batch_size} * ${num_gpus}'}"
222
+ - "num_gpus:${num_gpus}"
223
+ # model type could be unet or transformer
224
+ - "${model_type}"
225
+ # dataset file name (joined; substring-searchable on wandb)
226
+ # Note: full ${task.dataset_path} dropped — wandb tags max 64 chars.
227
+ - "${eval:'\"${task.dataset_path}\".split(\"/\")[-1].rsplit(\".\",1)[0]'}"
228
+ - "${mirror}"
229
+ # Only emit real_/sim_ tags when the dataset filename actually carries
230
+ # a real_<N> / sim_<N> token (otherwise the resolver returns None and we
231
+ # render an empty string here, which is filtered out before wandb init).
232
+ - "${eval:'\"real_size:%d\" % ${real_data_size} if ${real_data_size} is not None else \"\"'}"
233
+ - "${eval:'\"sim_size:%d\" % ${sim_data_size} if ${sim_data_size} is not None else \"\"'}"
234
+ - "${eval:'(\"real\" if ${real_data_size} > 0 else \"no_real\") if ${real_data_size} is not None else \"\"'}"
235
+ - "${eval:'(\"sim\" if ${sim_data_size} > 0 else \"no_sim\") if ${sim_data_size} is not None else \"\"'}"
236
+ - "${bbox}"
237
+ id: null
238
+ group: ${task_name}_${model_type}
239
+
240
+ checkpoint:
241
+ topk:
242
+ monitor_key: train_loss
243
+ mode: min
244
+ k: 20
245
+ format_str: '{task_name}-{model_type}-{visual_encoder}-batch_{total_batch}-{dataset_file_name}-epoch_{epoch:04d}-train_loss_{train_loss:.3f}.ckpt'
246
+ save_last_ckpt: True
247
+ save_last_snapshot: False
248
+
249
+ multi_run:
250
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
251
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
252
+
253
+ hydra:
254
+ job:
255
+ override_dirname: ${name}
256
+ run:
257
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
258
+ sweep:
259
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
260
+ subdir: ${hydra.job.num}