solarian-diffusion 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. diffusion_policy/__init__.py +0 -0
  2. diffusion_policy/codecs/__init__.py +0 -0
  3. diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
  4. diffusion_policy/common/__init__.py +0 -0
  5. diffusion_policy/common/checkpoint_util.py +64 -0
  6. diffusion_policy/common/json_logger.py +117 -0
  7. diffusion_policy/common/normalize_util.py +251 -0
  8. diffusion_policy/common/pose_repr_util.py +112 -0
  9. diffusion_policy/common/pytorch_util.py +82 -0
  10. diffusion_policy/common/q_critic_util.py +135 -0
  11. diffusion_policy/common/replay_buffer.py +672 -0
  12. diffusion_policy/common/sampler.py +204 -0
  13. diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
  14. diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
  15. diffusion_policy/config/legacy/task/can_image.yaml +64 -0
  16. diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
  17. diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
  18. diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
  19. diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
  20. diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
  21. diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
  22. diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
  23. diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
  24. diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
  25. diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
  26. diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
  27. diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
  28. diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
  29. diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
  30. diffusion_policy/config/legacy/task/square_image.yaml +64 -0
  31. diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
  32. diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
  33. diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
  34. diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
  35. diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
  36. diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
  37. diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
  38. diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
  39. diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
  40. diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
  41. diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
  42. diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
  43. diffusion_policy/config/legacy/task/umi.yaml +45 -0
  44. diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
  45. diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
  46. diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
  47. diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
  48. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
  49. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
  50. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
  51. diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
  52. diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
  53. diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
  54. diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
  55. diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
  56. diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
  57. diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
  58. diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
  59. diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
  60. diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
  61. diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
  62. diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
  63. diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
  64. diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
  65. diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
  66. diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
  67. diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
  68. diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
  69. diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
  70. diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
  71. diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
  72. diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
  73. diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
  74. diffusion_policy/config/rl_annotate.yaml +265 -0
  75. diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
  76. diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
  77. diffusion_policy/config/rl_annotate_v2.yaml +141 -0
  78. diffusion_policy/config/rl_annotate_v3.yaml +165 -0
  79. diffusion_policy/config/task/lift.yaml +89 -0
  80. diffusion_policy/config/task/square.yaml +92 -0
  81. diffusion_policy/config/task/tool_hang.yaml +93 -0
  82. diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
  83. diffusion_policy/config/task/umi.yaml +105 -0
  84. diffusion_policy/config/task/umi_bimanual.yaml +135 -0
  85. diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
  86. diffusion_policy/config/task/umi_image.yaml +36 -0
  87. diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
  88. diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
  89. diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
  90. diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
  91. diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
  92. diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
  93. diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
  94. diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
  95. diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
  96. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
  97. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
  98. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
  99. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
  100. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
  101. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
  102. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
  103. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
  104. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
  105. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
  106. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
  107. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
  108. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
  109. diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
  110. diffusion_policy/config/train_iql_umi.yaml +186 -0
  111. diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
  112. diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
  113. diffusion_policy/dataset/__init__.py +0 -0
  114. diffusion_policy/dataset/base_dataset.py +76 -0
  115. diffusion_policy/dataset/umi_dataset.py +652 -0
  116. diffusion_policy/env_runner/__init__.py +0 -0
  117. diffusion_policy/env_runner/base_image_runner.py +9 -0
  118. diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
  119. diffusion_policy/model/__init__.py +0 -0
  120. diffusion_policy/model/common/__init__.py +0 -0
  121. diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
  122. diffusion_policy/model/common/lr_scheduler.py +46 -0
  123. diffusion_policy/model/common/module_attr_mixin.py +14 -0
  124. diffusion_policy/model/common/normalizer.py +353 -0
  125. diffusion_policy/model/common/tensor_util.py +960 -0
  126. diffusion_policy/model/diffusion/__init__.py +0 -0
  127. diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
  128. diffusion_policy/model/diffusion/conv1d_components.py +46 -0
  129. diffusion_policy/model/diffusion/ema_model.py +88 -0
  130. diffusion_policy/model/diffusion/mask_generator.py +221 -0
  131. diffusion_policy/model/diffusion/positional_embedding.py +17 -0
  132. diffusion_policy/model/vision/__init__.py +0 -0
  133. diffusion_policy/model/vision/crop_randomizer.py +288 -0
  134. diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
  135. diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
  136. diffusion_policy/policy/__init__.py +0 -0
  137. diffusion_policy/policy/base_image_policy.py +27 -0
  138. diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
  139. diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
  140. diffusion_policy/policy/parl_action_optimizer.py +675 -0
  141. diffusion_policy/workspace/__init__.py +0 -0
  142. diffusion_policy/workspace/base_workspace.py +146 -0
  143. diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
  144. solarian_diffusion/__init__.py +32 -0
  145. solarian_diffusion/config.py +31 -0
  146. solarian_diffusion/policy_server.py +316 -0
  147. solarian_diffusion/rtc.py +272 -0
  148. solarian_diffusion/service.py +16 -0
  149. solarian_diffusion/trainer.py +213 -0
  150. solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
  151. solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
  152. solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
  153. solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
  154. solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
  155. solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
  156. solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
  157. umi/__init__.py +0 -0
  158. umi/common/__init__.py +0 -0
  159. umi/common/pose_util.py +115 -0
@@ -0,0 +1,265 @@
1
+ defaults:
2
+ - _self_
3
+ - task: umi_pick_single_lego_bbox
4
+
5
+ name: train_diffusion_unet_timm
6
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
7
+
8
+ # task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+ batch_size: 128
12
+ # visual_encoder: "vit_base_patch16_clip_224.openai"
13
+ # visual_encoder_tag: "vitb"
14
+ visual_encoder: "vit_large_patch14_clip_224"
15
+ visual_encoder_tag: "vitl"
16
+ # visual_encoder: "resnet34.a1_in1k"
17
+ # visual_encoder_tag: "resnet34"
18
+ model_type: "unet"
19
+ # this will be overridden at call time
20
+ num_gpus: 1
21
+
22
+
23
+ # Parsed from the dataset file name's hyphen-separated tokens.
24
+ # eg: pick_lego-black_mirror-real_200-sim_200-no_bbox.zarr.zip
25
+ # Resolver: train.py::parse_dataset_field. Last arg is the fallback default.
26
+ # mirror options: black_mirror, mirror, swap_mirror, no_mirror
27
+ mirror: ${parse_dataset_field:${task.dataset_path},mirror,no_mirror}
28
+ # real_data_size / sim_data_size are null when the filename has no real_<N>/sim_<N>
29
+ # token; the wandb tags below skip themselves in that case.
30
+ real_data_size: ${parse_dataset_field:${task.dataset_path},real_data_size,null}
31
+ sim_data_size: ${parse_dataset_field:${task.dataset_path},sim_data_size,null}
32
+ bbox: ${parse_dataset_field:${task.dataset_path},bbox,no_bbox}
33
+ # Surfaces the bbox integration mode (low_dim vs mask_channel) onto the
34
+ # wandb tag set so runs are filterable by it. Sourced from the task
35
+ # shape_meta so it tracks the actual model wiring, not a separate flag.
36
+ bbox_mode: ${task.shape_meta.obs.bbox.bbox_mode}
37
+ task_name: ${parse_dataset_field:${task.dataset_path},data_task,pick_lego}
38
+
39
+ # Override dataset path for fine-tuning
40
+ # This will override the dataset_path in task/umi.yaml
41
+ # Uncomment and set to your new dataset path:
42
+ # task.dataset_path: "path/to/your/new/dataset.zarr.zip"
43
+
44
+ n_action_steps: 8
45
+
46
+ policy:
47
+ _target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
48
+
49
+ shape_meta: ${shape_meta}
50
+
51
+ noise_scheduler:
52
+ _target_: diffusers.DDIMScheduler
53
+ num_train_timesteps: 50
54
+ beta_start: 0.0001
55
+ beta_end: 0.02
56
+ # beta_schedule is important
57
+ # this is the best we found
58
+ beta_schedule: squaredcos_cap_v2
59
+ clip_sample: True
60
+ set_alpha_to_one: True
61
+ steps_offset: 0
62
+ prediction_type: epsilon # or sample
63
+
64
+ obs_encoder:
65
+ _target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
66
+ shape_meta: ${shape_meta}
67
+
68
+ ##### from scratch #####
69
+ # model_name: 'vit_base_patch16_224'
70
+ # model_name: 'resnet34'
71
+ # model_name: 'vit_tiny_patch16_224'
72
+ # model_name: 'efficientnet_b0'
73
+ # model_name: 'efficientnet_b3'
74
+ # pretrained: False
75
+ # frozen: False
76
+
77
+ ##### from scratch #####
78
+ # model_name: 'resnet34.a1_in1k'
79
+ model_name: ${visual_encoder}
80
+ # model_name: 'vit_large_patch14_clip_224' # Changed to match checkpoint cup_wild_vit_l_1img.ckpt (ViT-L with CLIP weights)
81
+ # model_name: 'convnext_base.clip_laion2b_augreg_ft_in12k'
82
+ pretrained: True
83
+ frozen: False
84
+
85
+
86
+ global_pool: ''
87
+
88
+ # 'avg' or 'max' or 'soft_attention' or 'spatial_embedding' or 'transformer' or 'attention_pool_2d'
89
+ feature_aggregation: 'attention_pool_2d'
90
+ # feature_aggregation: null
91
+ position_encording: 'sinusoidal' # 'learnable' or 'sinusoidal'. it only works for transformer
92
+
93
+ # it only works for resnet. 32 (7x7) or 16 (14x14)
94
+ downsample_ratio: 32
95
+
96
+ transforms:
97
+ - type: RandomCrop
98
+ ratio: 0.95
99
+ # RandomRotation is not compatible with BF16
100
+ # - _target_: torchvision.transforms.RandomRotation
101
+ # degrees:
102
+ # - -5.0
103
+ # - 5.0
104
+ # expand: false
105
+ - _target_: torchvision.transforms.ColorJitter
106
+ brightness: 0.3
107
+ contrast: 0.4
108
+ saturation: 0.5
109
+ hue: 0.08
110
+ # - _target_: torchvision.transforms.RandomPerspective
111
+ # distortion_scale: 0.5
112
+ # p: 1.0
113
+ # - _target_: torchvision.transforms.ElasticTransform
114
+ # alpha: 50.0
115
+ # sigma: 5.0
116
+ # - _target_: torchvision.transforms.RandomPerspective
117
+ # distortion_scale: 0.5
118
+ # p: 1.0
119
+ # - _target_: torchvision.transforms.ElasticTransform
120
+ # alpha: 50.0
121
+ # sigma: 5.0
122
+ # - _target_: torchvision.transforms.RandomCrop
123
+ # size: 192
124
+
125
+
126
+ use_group_norm: True
127
+ share_rgb_model: False
128
+ imagenet_norm: True
129
+
130
+ num_inference_steps: 16
131
+ obs_as_global_cond: True
132
+ diffusion_step_embed_dim: 128
133
+ down_dims: [256,512,1024]
134
+ kernel_size: 5
135
+ n_groups: 8
136
+ cond_predict_scale: True
137
+ input_pertub: 0.1 # reference: https://github.com/forever208/DDPM-IP
138
+ # train multiple diffusion steps per obs encoder forward pass
139
+ # doing this too much will cause instability!
140
+ train_diffusion_n_samples: 1
141
+
142
+ ema:
143
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
144
+ update_after_step: 0
145
+ inv_gamma: 1.0
146
+ power: 0.75
147
+ min_value: 0.0
148
+ max_value: 0.9999
149
+
150
+ dataloader:
151
+ batch_size: ${batch_size}
152
+ num_workers: 8
153
+ shuffle: True
154
+ pin_memory: True
155
+ persistent_workers: True
156
+ prefetch_factor: 4
157
+
158
+ val_dataloader:
159
+ batch_size: ${batch_size}
160
+ num_workers: 8
161
+ shuffle: False
162
+ pin_memory: True
163
+ persistent_workers: True
164
+ prefetch_factor: 4
165
+
166
+ # dataloader:
167
+ # batch_size: 64
168
+ # num_workers: 16
169
+ # shuffle: True
170
+ # pin_memory: True
171
+ # persistent_workers: True
172
+
173
+ # val_dataloader:
174
+ # batch_size: 64
175
+ # num_workers: 16
176
+ # shuffle: False
177
+ # pin_memory: True
178
+ # persistent_workers: True
179
+
180
+ optimizer:
181
+ _target_: torch.optim.AdamW
182
+ lr: 3.0e-4
183
+ betas: [0.95, 0.999]
184
+ eps: 1.0e-8
185
+ weight_decay: 1.0e-6
186
+
187
+ training:
188
+ device: "cuda:0"
189
+ seed: 42
190
+ debug: False
191
+ resume: False
192
+ # optimization
193
+ lr_scheduler: cosine
194
+ lr_warmup_steps: 500
195
+ num_epochs: 250
196
+ gradient_accumulate_every: 1
197
+ # EMA destroys performance when used with BatchNorm
198
+ # replace BatchNorm with GroupNorm.
199
+ use_ema: True
200
+ freeze_encoder: False
201
+ # training loop control
202
+ # in epochs
203
+ rollout_every: 10
204
+ checkpoint_every: 10
205
+ val_every: 1
206
+ sample_every: 5
207
+ # steps per epoch
208
+ max_train_steps: null
209
+ max_val_steps: null
210
+ # misc
211
+ tqdm_interval_sec: 1.0
212
+
213
+ logging:
214
+ project: umi
215
+ resume: False
216
+ mode: online
217
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
218
+ tags:
219
+ - "${name}"
220
+ - "${task_name}"
221
+ - "${exp_name}"
222
+ # visual encoder could be vit_base_patch16_clip_224.openai, resnet34, etc.
223
+ - "${visual_encoder_tag}"
224
+ # batch size for each gpu
225
+ - "batch_size:${eval:'${batch_size} * ${num_gpus}'}"
226
+ - "num_gpus:${num_gpus}"
227
+ # model type could be unet or transformer
228
+ - "${model_type}"
229
+ # dataset file name (joined; substring-searchable on wandb)
230
+ # Note: full ${task.dataset_path} dropped — wandb tags max 64 chars.
231
+ - "${eval:'\"${task.dataset_path}\".split(\"/\")[-1].rsplit(\".\",1)[0]'}"
232
+ - "${mirror}"
233
+ # Only emit real_/sim_ tags when the dataset filename actually carries
234
+ # a real_<N> / sim_<N> token (otherwise the resolver returns None and we
235
+ # render an empty string here, which is filtered out before wandb init).
236
+ - "${eval:'\"real_size:%d\" % ${real_data_size} if ${real_data_size} is not None else \"\"'}"
237
+ - "${eval:'\"sim_size:%d\" % ${sim_data_size} if ${sim_data_size} is not None else \"\"'}"
238
+ - "${eval:'(\"real\" if ${real_data_size} > 0 else \"no_real\") if ${real_data_size} is not None else \"\"'}"
239
+ - "${eval:'(\"sim\" if ${sim_data_size} > 0 else \"no_sim\") if ${sim_data_size} is not None else \"\"'}"
240
+ - "${bbox}"
241
+ - "bbox_mode:${bbox_mode}"
242
+ id: null
243
+ group: ${task_name}_${model_type}
244
+
245
+ checkpoint:
246
+ topk:
247
+ monitor_key: train_loss
248
+ mode: min
249
+ k: 20
250
+ format_str: '{task_name}-{model_type}-{visual_encoder}-batch_{total_batch}-{dataset_file_name}-epoch_{epoch:04d}-train_loss_{train_loss:.3f}.ckpt'
251
+ save_last_ckpt: True
252
+ save_last_snapshot: False
253
+
254
+ multi_run:
255
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
256
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
257
+
258
+ hydra:
259
+ job:
260
+ override_dirname: ${name}
261
+ run:
262
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
263
+ sweep:
264
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
265
+ subdir: ${hydra.job.num}
@@ -0,0 +1,265 @@
1
+ defaults:
2
+ - _self_
3
+ - task: umi_pick_single_lego_bbox
4
+
5
+ name: train_diffusion_unet_timm
6
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
7
+
8
+ # task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+ batch_size: 128
12
+ # visual_encoder: "vit_base_patch16_clip_224.openai"
13
+ # visual_encoder_tag: "vitb"
14
+ visual_encoder: "vit_large_patch14_clip_224"
15
+ visual_encoder_tag: "vitl"
16
+ # visual_encoder: "resnet34.a1_in1k"
17
+ # visual_encoder_tag: "resnet34"
18
+ model_type: "unet"
19
+ # this will be overridden at call time
20
+ num_gpus: 1
21
+
22
+
23
+ # Parsed from the dataset file name's hyphen-separated tokens.
24
+ # eg: pick_lego-black_mirror-real_200-sim_200-no_bbox.zarr.zip
25
+ # Resolver: train.py::parse_dataset_field. Last arg is the fallback default.
26
+ # mirror options: black_mirror, mirror, swap_mirror, no_mirror
27
+ mirror: ${parse_dataset_field:${task.dataset_path},mirror,no_mirror}
28
+ # real_data_size / sim_data_size are null when the filename has no real_<N>/sim_<N>
29
+ # token; the wandb tags below skip themselves in that case.
30
+ real_data_size: ${parse_dataset_field:${task.dataset_path},real_data_size,null}
31
+ sim_data_size: ${parse_dataset_field:${task.dataset_path},sim_data_size,null}
32
+ bbox: ${parse_dataset_field:${task.dataset_path},bbox,no_bbox}
33
+ # Surfaces the bbox integration mode (low_dim vs mask_channel) onto the
34
+ # wandb tag set so runs are filterable by it. Sourced from the task
35
+ # shape_meta so it tracks the actual model wiring, not a separate flag.
36
+ bbox_mode: ${task.shape_meta.obs.bbox.bbox_mode}
37
+ task_name: ${parse_dataset_field:${task.dataset_path},data_task,pick_lego}
38
+
39
+ # Override dataset path for fine-tuning
40
+ # This will override the dataset_path in task/umi.yaml
41
+ # Uncomment and set to your new dataset path:
42
+ # task.dataset_path: "path/to/your/new/dataset.zarr.zip"
43
+
44
+ n_action_steps: 8
45
+
46
+ policy:
47
+ _target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
48
+
49
+ shape_meta: ${shape_meta}
50
+
51
+ noise_scheduler:
52
+ _target_: diffusers.DDIMScheduler
53
+ num_train_timesteps: 50
54
+ beta_start: 0.0001
55
+ beta_end: 0.02
56
+ # beta_schedule is important
57
+ # this is the best we found
58
+ beta_schedule: squaredcos_cap_v2
59
+ clip_sample: True
60
+ set_alpha_to_one: True
61
+ steps_offset: 0
62
+ prediction_type: epsilon # or sample
63
+
64
+ obs_encoder:
65
+ _target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
66
+ shape_meta: ${shape_meta}
67
+
68
+ ##### from scratch #####
69
+ # model_name: 'vit_base_patch16_224'
70
+ # model_name: 'resnet34'
71
+ # model_name: 'vit_tiny_patch16_224'
72
+ # model_name: 'efficientnet_b0'
73
+ # model_name: 'efficientnet_b3'
74
+ # pretrained: False
75
+ # frozen: False
76
+
77
+ ##### from scratch #####
78
+ # model_name: 'resnet34.a1_in1k'
79
+ model_name: ${visual_encoder}
80
+ # model_name: 'vit_large_patch14_clip_224' # Changed to match checkpoint cup_wild_vit_l_1img.ckpt (ViT-L with CLIP weights)
81
+ # model_name: 'convnext_base.clip_laion2b_augreg_ft_in12k'
82
+ pretrained: True
83
+ frozen: False
84
+
85
+
86
+ global_pool: ''
87
+
88
+ # 'avg' or 'max' or 'soft_attention' or 'spatial_embedding' or 'transformer' or 'attention_pool_2d'
89
+ feature_aggregation: 'attention_pool_2d'
90
+ # feature_aggregation: null
91
+ position_encording: 'sinusoidal' # 'learnable' or 'sinusoidal'. it only works for transformer
92
+
93
+ # it only works for resnet. 32 (7x7) or 16 (14x14)
94
+ downsample_ratio: 32
95
+
96
+ transforms:
97
+ - type: RandomCrop
98
+ ratio: 0.95
99
+ # RandomRotation is not compatible with BF16
100
+ # - _target_: torchvision.transforms.RandomRotation
101
+ # degrees:
102
+ # - -5.0
103
+ # - 5.0
104
+ # expand: false
105
+ - _target_: torchvision.transforms.ColorJitter
106
+ brightness: 0.3
107
+ contrast: 0.4
108
+ saturation: 0.5
109
+ hue: 0.08
110
+ # - _target_: torchvision.transforms.RandomPerspective
111
+ # distortion_scale: 0.5
112
+ # p: 1.0
113
+ # - _target_: torchvision.transforms.ElasticTransform
114
+ # alpha: 50.0
115
+ # sigma: 5.0
116
+ # - _target_: torchvision.transforms.RandomPerspective
117
+ # distortion_scale: 0.5
118
+ # p: 1.0
119
+ # - _target_: torchvision.transforms.ElasticTransform
120
+ # alpha: 50.0
121
+ # sigma: 5.0
122
+ # - _target_: torchvision.transforms.RandomCrop
123
+ # size: 192
124
+
125
+
126
+ use_group_norm: True
127
+ share_rgb_model: False
128
+ imagenet_norm: True
129
+
130
+ num_inference_steps: 16
131
+ obs_as_global_cond: True
132
+ diffusion_step_embed_dim: 128
133
+ down_dims: [256,512,1024]
134
+ kernel_size: 5
135
+ n_groups: 8
136
+ cond_predict_scale: True
137
+ input_pertub: 0.1 # reference: https://github.com/forever208/DDPM-IP
138
+ # train multiple diffusion steps per obs encoder forward pass
139
+ # doing this too much will cause instability!
140
+ train_diffusion_n_samples: 1
141
+
142
+ ema:
143
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
144
+ update_after_step: 0
145
+ inv_gamma: 1.0
146
+ power: 0.75
147
+ min_value: 0.0
148
+ max_value: 0.9999
149
+
150
+ dataloader:
151
+ batch_size: ${batch_size}
152
+ num_workers: 8
153
+ shuffle: True
154
+ pin_memory: True
155
+ persistent_workers: True
156
+ prefetch_factor: 4
157
+
158
+ val_dataloader:
159
+ batch_size: ${batch_size}
160
+ num_workers: 8
161
+ shuffle: False
162
+ pin_memory: True
163
+ persistent_workers: True
164
+ prefetch_factor: 4
165
+
166
+ # dataloader:
167
+ # batch_size: 64
168
+ # num_workers: 16
169
+ # shuffle: True
170
+ # pin_memory: True
171
+ # persistent_workers: True
172
+
173
+ # val_dataloader:
174
+ # batch_size: 64
175
+ # num_workers: 16
176
+ # shuffle: False
177
+ # pin_memory: True
178
+ # persistent_workers: True
179
+
180
+ optimizer:
181
+ _target_: torch.optim.AdamW
182
+ lr: 3.0e-4
183
+ betas: [0.95, 0.999]
184
+ eps: 1.0e-8
185
+ weight_decay: 1.0e-6
186
+
187
+ training:
188
+ device: "cuda:0"
189
+ seed: 42
190
+ debug: False
191
+ resume: False
192
+ # optimization
193
+ lr_scheduler: cosine
194
+ lr_warmup_steps: 500
195
+ num_epochs: 250
196
+ gradient_accumulate_every: 1
197
+ # EMA destroys performance when used with BatchNorm
198
+ # replace BatchNorm with GroupNorm.
199
+ use_ema: True
200
+ freeze_encoder: False
201
+ # training loop control
202
+ # in epochs
203
+ rollout_every: 10
204
+ checkpoint_every: 10
205
+ val_every: 1
206
+ sample_every: 5
207
+ # steps per epoch
208
+ max_train_steps: null
209
+ max_val_steps: null
210
+ # misc
211
+ tqdm_interval_sec: 1.0
212
+
213
+ logging:
214
+ project: umi
215
+ resume: False
216
+ mode: online
217
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
218
+ tags:
219
+ - "${name}"
220
+ - "${task_name}"
221
+ - "${exp_name}"
222
+ # visual encoder could be vit_base_patch16_clip_224.openai, resnet34, etc.
223
+ - "${visual_encoder_tag}"
224
+ # batch size for each gpu
225
+ - "batch_size:${eval:'${batch_size} * ${num_gpus}'}"
226
+ - "num_gpus:${num_gpus}"
227
+ # model type could be unet or transformer
228
+ - "${model_type}"
229
+ # dataset file name (joined; substring-searchable on wandb)
230
+ # Note: full ${task.dataset_path} dropped — wandb tags max 64 chars.
231
+ - "${eval:'\"${task.dataset_path}\".split(\"/\")[-1].rsplit(\".\",1)[0]'}"
232
+ - "${mirror}"
233
+ # Only emit real_/sim_ tags when the dataset filename actually carries
234
+ # a real_<N> / sim_<N> token (otherwise the resolver returns None and we
235
+ # render an empty string here, which is filtered out before wandb init).
236
+ - "${eval:'\"real_size:%d\" % ${real_data_size} if ${real_data_size} is not None else \"\"'}"
237
+ - "${eval:'\"sim_size:%d\" % ${sim_data_size} if ${sim_data_size} is not None else \"\"'}"
238
+ - "${eval:'(\"real\" if ${real_data_size} > 0 else \"no_real\") if ${real_data_size} is not None else \"\"'}"
239
+ - "${eval:'(\"sim\" if ${sim_data_size} > 0 else \"no_sim\") if ${sim_data_size} is not None else \"\"'}"
240
+ - "${bbox}"
241
+ - "bbox_mode:${bbox_mode}"
242
+ id: null
243
+ group: ${task_name}_${model_type}
244
+
245
+ checkpoint:
246
+ topk:
247
+ monitor_key: train_loss
248
+ mode: min
249
+ k: 20
250
+ format_str: '{task_name}-{model_type}-{visual_encoder}-batch_{total_batch}-{dataset_file_name}-epoch_{epoch:04d}-train_loss_{train_loss:.3f}.ckpt'
251
+ save_last_ckpt: True
252
+ save_last_snapshot: False
253
+
254
+ multi_run:
255
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
256
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
257
+
258
+ hydra:
259
+ job:
260
+ override_dirname: ${name}
261
+ run:
262
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
263
+ sweep:
264
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
265
+ subdir: ${hydra.job.num}