solarian-diffusion 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. diffusion_policy/__init__.py +0 -0
  2. diffusion_policy/codecs/__init__.py +0 -0
  3. diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
  4. diffusion_policy/common/__init__.py +0 -0
  5. diffusion_policy/common/checkpoint_util.py +64 -0
  6. diffusion_policy/common/json_logger.py +117 -0
  7. diffusion_policy/common/normalize_util.py +251 -0
  8. diffusion_policy/common/pose_repr_util.py +112 -0
  9. diffusion_policy/common/pytorch_util.py +82 -0
  10. diffusion_policy/common/q_critic_util.py +135 -0
  11. diffusion_policy/common/replay_buffer.py +672 -0
  12. diffusion_policy/common/sampler.py +204 -0
  13. diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
  14. diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
  15. diffusion_policy/config/legacy/task/can_image.yaml +64 -0
  16. diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
  17. diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
  18. diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
  19. diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
  20. diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
  21. diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
  22. diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
  23. diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
  24. diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
  25. diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
  26. diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
  27. diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
  28. diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
  29. diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
  30. diffusion_policy/config/legacy/task/square_image.yaml +64 -0
  31. diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
  32. diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
  33. diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
  34. diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
  35. diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
  36. diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
  37. diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
  38. diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
  39. diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
  40. diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
  41. diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
  42. diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
  43. diffusion_policy/config/legacy/task/umi.yaml +45 -0
  44. diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
  45. diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
  46. diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
  47. diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
  48. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
  49. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
  50. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
  51. diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
  52. diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
  53. diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
  54. diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
  55. diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
  56. diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
  57. diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
  58. diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
  59. diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
  60. diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
  61. diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
  62. diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
  63. diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
  64. diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
  65. diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
  66. diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
  67. diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
  68. diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
  69. diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
  70. diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
  71. diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
  72. diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
  73. diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
  74. diffusion_policy/config/rl_annotate.yaml +265 -0
  75. diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
  76. diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
  77. diffusion_policy/config/rl_annotate_v2.yaml +141 -0
  78. diffusion_policy/config/rl_annotate_v3.yaml +165 -0
  79. diffusion_policy/config/task/lift.yaml +89 -0
  80. diffusion_policy/config/task/square.yaml +92 -0
  81. diffusion_policy/config/task/tool_hang.yaml +93 -0
  82. diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
  83. diffusion_policy/config/task/umi.yaml +105 -0
  84. diffusion_policy/config/task/umi_bimanual.yaml +135 -0
  85. diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
  86. diffusion_policy/config/task/umi_image.yaml +36 -0
  87. diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
  88. diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
  89. diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
  90. diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
  91. diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
  92. diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
  93. diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
  94. diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
  95. diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
  96. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
  97. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
  98. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
  99. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
  100. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
  101. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
  102. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
  103. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
  104. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
  105. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
  106. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
  107. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
  108. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
  109. diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
  110. diffusion_policy/config/train_iql_umi.yaml +186 -0
  111. diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
  112. diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
  113. diffusion_policy/dataset/__init__.py +0 -0
  114. diffusion_policy/dataset/base_dataset.py +76 -0
  115. diffusion_policy/dataset/umi_dataset.py +652 -0
  116. diffusion_policy/env_runner/__init__.py +0 -0
  117. diffusion_policy/env_runner/base_image_runner.py +9 -0
  118. diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
  119. diffusion_policy/model/__init__.py +0 -0
  120. diffusion_policy/model/common/__init__.py +0 -0
  121. diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
  122. diffusion_policy/model/common/lr_scheduler.py +46 -0
  123. diffusion_policy/model/common/module_attr_mixin.py +14 -0
  124. diffusion_policy/model/common/normalizer.py +353 -0
  125. diffusion_policy/model/common/tensor_util.py +960 -0
  126. diffusion_policy/model/diffusion/__init__.py +0 -0
  127. diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
  128. diffusion_policy/model/diffusion/conv1d_components.py +46 -0
  129. diffusion_policy/model/diffusion/ema_model.py +88 -0
  130. diffusion_policy/model/diffusion/mask_generator.py +221 -0
  131. diffusion_policy/model/diffusion/positional_embedding.py +17 -0
  132. diffusion_policy/model/vision/__init__.py +0 -0
  133. diffusion_policy/model/vision/crop_randomizer.py +288 -0
  134. diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
  135. diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
  136. diffusion_policy/policy/__init__.py +0 -0
  137. diffusion_policy/policy/base_image_policy.py +27 -0
  138. diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
  139. diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
  140. diffusion_policy/policy/parl_action_optimizer.py +675 -0
  141. diffusion_policy/workspace/__init__.py +0 -0
  142. diffusion_policy/workspace/base_workspace.py +146 -0
  143. diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
  144. solarian_diffusion/__init__.py +32 -0
  145. solarian_diffusion/config.py +31 -0
  146. solarian_diffusion/policy_server.py +316 -0
  147. solarian_diffusion/rtc.py +272 -0
  148. solarian_diffusion/service.py +16 -0
  149. solarian_diffusion/trainer.py +213 -0
  150. solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
  151. solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
  152. solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
  153. solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
  154. solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
  155. solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
  156. solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
  157. umi/__init__.py +0 -0
  158. umi/common/__init__.py +0 -0
  159. umi/common/pose_util.py +115 -0
@@ -0,0 +1,224 @@
1
+ defaults:
2
+ - _self_
3
+ - task: umi
4
+
5
+ name: train_diffusion_unet_timm
6
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
7
+
8
+ task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+
12
+ # Override dataset path for fine-tuning
13
+ # This will override the dataset_path in task/umi.yaml
14
+ # Uncomment and set to your new dataset path:
15
+ # task.dataset_path: "path/to/your/new/dataset.zarr.zip"
16
+
17
+ n_action_steps: 8
18
+
19
+ policy:
20
+ _target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
21
+
22
+ shape_meta: ${shape_meta}
23
+
24
+ noise_scheduler:
25
+ _target_: diffusers.DDIMScheduler
26
+ num_train_timesteps: 50
27
+ beta_start: 0.0001
28
+ beta_end: 0.02
29
+ # beta_schedule is important
30
+ # this is the best we found
31
+ beta_schedule: squaredcos_cap_v2
32
+ clip_sample: True
33
+ set_alpha_to_one: True
34
+ steps_offset: 0
35
+ prediction_type: epsilon # or sample
36
+
37
+ obs_encoder:
38
+ _target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
39
+ shape_meta: ${shape_meta}
40
+
41
+ ##### from scratch #####
42
+ # model_name: 'vit_base_patch16_224'
43
+ # model_name: 'resnet34'
44
+ # model_name: 'vit_tiny_patch16_224'
45
+ # model_name: 'efficientnet_b0'
46
+ # model_name: 'efficientnet_b3'
47
+ # pretrained: False
48
+ # frozen: False
49
+
50
+ ##### from scratch #####
51
+ # model_name: 'resnet34.a1_in1k'
52
+ # model_name: 'vit_base_patch16_clip_224.openai'
53
+ model_name: 'vit_large_patch14_clip_224' # Changed to match checkpoint cup_wild_vit_l_1img.ckpt (ViT-L with CLIP weights)
54
+ # model_name: 'convnext_base.clip_laion2b_augreg_ft_in12k'
55
+ pretrained: True
56
+ frozen: False
57
+
58
+
59
+ global_pool: ''
60
+
61
+ # 'avg' or 'max' or 'soft_attention' or 'spatial_embedding' or 'transformer' or 'attention_pool_2d'
62
+ feature_aggregation: 'attention_pool_2d'
63
+ # feature_aggregation: null
64
+ position_encording: 'sinusoidal' # 'learnable' or 'sinusoidal'. it only works for transformer
65
+
66
+ # it only works for resnet. 32 (7x7) or 16 (14x14)
67
+ downsample_ratio: 32
68
+
69
+ transforms:
70
+ - type: RandomCrop
71
+ ratio: 0.95
72
+ # RandomRotation is not compatible with BF16
73
+ # - _target_: torchvision.transforms.RandomRotation
74
+ # degrees:
75
+ # - -5.0
76
+ # - 5.0
77
+ # expand: false
78
+ - _target_: torchvision.transforms.ColorJitter
79
+ brightness: 0.3
80
+ contrast: 0.4
81
+ saturation: 0.5
82
+ hue: 0.08
83
+ # - _target_: torchvision.transforms.RandomPerspective
84
+ # distortion_scale: 0.5
85
+ # p: 1.0
86
+ # - _target_: torchvision.transforms.ElasticTransform
87
+ # alpha: 50.0
88
+ # sigma: 5.0
89
+ # - _target_: torchvision.transforms.RandomPerspective
90
+ # distortion_scale: 0.5
91
+ # p: 1.0
92
+ # - _target_: torchvision.transforms.ElasticTransform
93
+ # alpha: 50.0
94
+ # sigma: 5.0
95
+ # - _target_: torchvision.transforms.RandomCrop
96
+ # size: 192
97
+
98
+
99
+ use_group_norm: True
100
+ share_rgb_model: False
101
+ imagenet_norm: True
102
+
103
+ num_inference_steps: 16
104
+ obs_as_global_cond: True
105
+ diffusion_step_embed_dim: 128
106
+ down_dims: [256,512,1024]
107
+ kernel_size: 5
108
+ n_groups: 8
109
+ cond_predict_scale: True
110
+ input_pertub: 0.1 # reference: https://github.com/forever208/DDPM-IP
111
+ # train multiple diffusion steps per obs encoder forward pass
112
+ # doing this too much will cause instability!
113
+ train_diffusion_n_samples: 1
114
+
115
+ ema:
116
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
117
+ update_after_step: 0
118
+ inv_gamma: 1.0
119
+ power: 0.75
120
+ min_value: 0.0
121
+ max_value: 0.9999
122
+
123
+ dataloader:
124
+ # NOTE: with accelerate multi-GPU, a previous run produced a
125
+ # "DataLoader will create 32 worker processes in total" warning at
126
+ # num_workers=8 (probably 8 * num_processes from accelerate's
127
+ # prepare_data_loader). Plus persistent_workers x prefetch_factor x N ranks
128
+ # was the leading suspect for the OOM-kill at epoch 54. Keep this modest;
129
+ # the bottleneck is GPU, not CPU decode.
130
+ batch_size: 64
131
+ num_workers: 4
132
+ shuffle: True
133
+ pin_memory: True
134
+ persistent_workers: True
135
+
136
+ val_dataloader:
137
+ # Validation is small (~5k samples, 40 batches) and runs once per epoch
138
+ # on the main process only. No need for persistent workers eating RAM
139
+ # between epochs, and 2 workers easily saturates the eval pass.
140
+ batch_size: 64
141
+ num_workers: 2
142
+ shuffle: False
143
+ pin_memory: True
144
+ persistent_workers: False
145
+
146
+ # dataloader:
147
+ # batch_size: 64
148
+ # num_workers: 16
149
+ # shuffle: True
150
+ # pin_memory: True
151
+ # persistent_workers: True
152
+
153
+ # val_dataloader:
154
+ # batch_size: 64
155
+ # num_workers: 16
156
+ # shuffle: False
157
+ # pin_memory: True
158
+ # persistent_workers: True
159
+
160
+ optimizer:
161
+ _target_: torch.optim.AdamW
162
+ lr: 3.0e-4
163
+ betas: [0.95, 0.999]
164
+ eps: 1.0e-8
165
+ weight_decay: 1.0e-6
166
+
167
+ training:
168
+ device: "cuda:0"
169
+ seed: 42
170
+ debug: False
171
+ resume: False
172
+ # optimization
173
+ lr_scheduler: cosine
174
+ lr_warmup_steps: 2000
175
+ num_epochs: 120
176
+ gradient_accumulate_every: 1
177
+ # EMA destroys performance when used with BatchNorm
178
+ # replace BatchNorm with GroupNorm.
179
+ use_ema: True
180
+ freeze_encoder: False
181
+ # training loop control
182
+ # in epochs
183
+ rollout_every: 10
184
+ # Cut to 5 so a future hard-kill loses at most 5 epochs (~10 min),
185
+ # not 10 (~20 min). TopK manager still keeps the best ckpts.
186
+ checkpoint_every: 5
187
+ val_every: 1
188
+ sample_every: 5
189
+ # steps per epoch
190
+ max_train_steps: null
191
+ max_val_steps: null
192
+ # misc
193
+ tqdm_interval_sec: 1.0
194
+
195
+ logging:
196
+ project: umi
197
+ resume: False
198
+ mode: online
199
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
200
+ tags: ["${name}", "${task_name}", "${exp_name}"]
201
+ id: null
202
+ group: null
203
+
204
+ checkpoint:
205
+ topk:
206
+ monitor_key: train_loss
207
+ mode: min
208
+ k: 20
209
+ format_str: 'epoch_{epoch:04d}-train_loss_{train_loss:.3f}.ckpt'
210
+ save_last_ckpt: True
211
+ save_last_snapshot: False
212
+
213
+ multi_run:
214
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
215
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
216
+
217
+ hydra:
218
+ job:
219
+ override_dirname: ${name}
220
+ run:
221
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
222
+ sweep:
223
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
224
+ subdir: ${hydra.job.num}
@@ -0,0 +1,213 @@
1
+ defaults:
2
+ - _self_
3
+ - task: umi
4
+
5
+ name: train_diffusion_unet_timm
6
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
7
+
8
+ task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+
12
+ # Override dataset path for fine-tuning
13
+ # This will override the dataset_path in task/umi.yaml
14
+ # Uncomment and set to your new dataset path:
15
+ # task.dataset_path: "path/to/your/new/dataset.zarr.zip"
16
+
17
+ n_action_steps: 8
18
+
19
+ policy:
20
+ _target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
21
+
22
+ shape_meta: ${shape_meta}
23
+
24
+ noise_scheduler:
25
+ _target_: diffusers.DDIMScheduler
26
+ num_train_timesteps: 50
27
+ beta_start: 0.0001
28
+ beta_end: 0.02
29
+ # beta_schedule is important
30
+ # this is the best we found
31
+ beta_schedule: squaredcos_cap_v2
32
+ clip_sample: True
33
+ set_alpha_to_one: True
34
+ steps_offset: 0
35
+ prediction_type: epsilon # or sample
36
+
37
+ obs_encoder:
38
+ _target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
39
+ shape_meta: ${shape_meta}
40
+
41
+ ##### from scratch #####
42
+ # model_name: 'vit_base_patch16_224'
43
+ # model_name: 'resnet34'
44
+ # model_name: 'vit_tiny_patch16_224'
45
+ # model_name: 'efficientnet_b0'
46
+ # model_name: 'efficientnet_b3'
47
+ # pretrained: False
48
+ # frozen: False
49
+
50
+ ##### from scratch #####
51
+ # model_name: 'resnet34.a1_in1k'
52
+ model_name: 'vit_base_patch16_clip_224.openai'
53
+ # model_name: 'vit_large_patch14_clip_224' # Changed to match checkpoint cup_wild_vit_l_1img.ckpt (ViT-L with CLIP weights)
54
+ # model_name: 'convnext_base.clip_laion2b_augreg_ft_in12k'
55
+ pretrained: True
56
+ frozen: False
57
+
58
+
59
+ global_pool: ''
60
+
61
+ # 'avg' or 'max' or 'soft_attention' or 'spatial_embedding' or 'transformer' or 'attention_pool_2d'
62
+ feature_aggregation: 'attention_pool_2d'
63
+ # feature_aggregation: null
64
+ position_encording: 'sinusoidal' # 'learnable' or 'sinusoidal'. it only works for transformer
65
+
66
+ # it only works for resnet. 32 (7x7) or 16 (14x14)
67
+ downsample_ratio: 32
68
+
69
+ transforms:
70
+ - type: RandomCrop
71
+ ratio: 0.95
72
+ # RandomRotation is not compatible with BF16
73
+ # - _target_: torchvision.transforms.RandomRotation
74
+ # degrees:
75
+ # - -5.0
76
+ # - 5.0
77
+ # expand: false
78
+ - _target_: torchvision.transforms.ColorJitter
79
+ brightness: 0.3
80
+ contrast: 0.4
81
+ saturation: 0.5
82
+ hue: 0.08
83
+ # - _target_: torchvision.transforms.RandomPerspective
84
+ # distortion_scale: 0.5
85
+ # p: 1.0
86
+ # - _target_: torchvision.transforms.ElasticTransform
87
+ # alpha: 50.0
88
+ # sigma: 5.0
89
+ # - _target_: torchvision.transforms.RandomPerspective
90
+ # distortion_scale: 0.5
91
+ # p: 1.0
92
+ # - _target_: torchvision.transforms.ElasticTransform
93
+ # alpha: 50.0
94
+ # sigma: 5.0
95
+ # - _target_: torchvision.transforms.RandomCrop
96
+ # size: 192
97
+
98
+
99
+ use_group_norm: True
100
+ share_rgb_model: False
101
+ imagenet_norm: True
102
+
103
+ num_inference_steps: 16
104
+ obs_as_global_cond: True
105
+ diffusion_step_embed_dim: 128
106
+ down_dims: [256,512,1024]
107
+ kernel_size: 5
108
+ n_groups: 8
109
+ cond_predict_scale: True
110
+ input_pertub: 0.1 # reference: https://github.com/forever208/DDPM-IP
111
+ # train multiple diffusion steps per obs encoder forward pass
112
+ # doing this too much will cause instability!
113
+ train_diffusion_n_samples: 1
114
+
115
+ ema:
116
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
117
+ update_after_step: 0
118
+ inv_gamma: 1.0
119
+ power: 0.75
120
+ min_value: 0.0
121
+ max_value: 0.9999
122
+
123
+ dataloader:
124
+ batch_size: 128
125
+ num_workers: 16
126
+ shuffle: True
127
+ pin_memory: True
128
+ persistent_workers: True
129
+
130
+ val_dataloader:
131
+ batch_size: 128
132
+ num_workers: 16
133
+ shuffle: False
134
+ pin_memory: True
135
+ persistent_workers: True
136
+
137
+ # dataloader:
138
+ # batch_size: 64
139
+ # num_workers: 16
140
+ # shuffle: True
141
+ # pin_memory: True
142
+ # persistent_workers: True
143
+
144
+ # val_dataloader:
145
+ # batch_size: 64
146
+ # num_workers: 16
147
+ # shuffle: False
148
+ # pin_memory: True
149
+ # persistent_workers: True
150
+
151
+ optimizer:
152
+ _target_: torch.optim.AdamW
153
+ lr: 3.0e-4
154
+ betas: [0.95, 0.999]
155
+ eps: 1.0e-8
156
+ weight_decay: 1.0e-6
157
+
158
+ training:
159
+ device: "cuda:0"
160
+ seed: 42
161
+ debug: False
162
+ resume: False
163
+ # optimization
164
+ lr_scheduler: cosine
165
+ lr_warmup_steps: 2000
166
+ num_epochs: 250
167
+ gradient_accumulate_every: 1
168
+ # EMA destroys performance when used with BatchNorm
169
+ # replace BatchNorm with GroupNorm.
170
+ use_ema: True
171
+ freeze_encoder: False
172
+ # training loop control
173
+ # in epochs
174
+ rollout_every: 10
175
+ checkpoint_every: 10
176
+ val_every: 1
177
+ sample_every: 5
178
+ # steps per epoch
179
+ max_train_steps: null
180
+ max_val_steps: null
181
+ # misc
182
+ tqdm_interval_sec: 1.0
183
+
184
+ logging:
185
+ project: umi
186
+ resume: False
187
+ mode: online
188
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
189
+ tags: ["${name}", "${task_name}", "${exp_name}"]
190
+ id: null
191
+ group: null
192
+
193
+ checkpoint:
194
+ topk:
195
+ monitor_key: train_loss
196
+ mode: min
197
+ k: 20
198
+ format_str: 'epoch_{epoch:04d}-train_loss_{train_loss:.3f}.ckpt'
199
+ save_last_ckpt: True
200
+ save_last_snapshot: False
201
+
202
+ multi_run:
203
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
204
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
205
+
206
+ hydra:
207
+ job:
208
+ override_dirname: ${name}
209
+ run:
210
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
211
+ sweep:
212
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
213
+ subdir: ${hydra.job.num}
@@ -0,0 +1,213 @@
1
+ defaults:
2
+ - _self_
3
+ - task: umi
4
+
5
+ name: train_diffusion_unet_timm
6
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
7
+
8
+ task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+
12
+ # Override dataset path for fine-tuning
13
+ # This will override the dataset_path in task/umi.yaml
14
+ # Uncomment and set to your new dataset path:
15
+ # task.dataset_path: "path/to/your/new/dataset.zarr.zip"
16
+
17
+ n_action_steps: 8
18
+
19
+ policy:
20
+ _target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
21
+
22
+ shape_meta: ${shape_meta}
23
+
24
+ noise_scheduler:
25
+ _target_: diffusers.DDIMScheduler
26
+ num_train_timesteps: 50
27
+ beta_start: 0.0001
28
+ beta_end: 0.02
29
+ # beta_schedule is important
30
+ # this is the best we found
31
+ beta_schedule: squaredcos_cap_v2
32
+ clip_sample: True
33
+ set_alpha_to_one: True
34
+ steps_offset: 0
35
+ prediction_type: epsilon # or sample
36
+
37
+ obs_encoder:
38
+ _target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
39
+ shape_meta: ${shape_meta}
40
+
41
+ ##### from scratch #####
42
+ # model_name: 'vit_base_patch16_224'
43
+ # model_name: 'resnet34'
44
+ # model_name: 'vit_tiny_patch16_224'
45
+ # model_name: 'efficientnet_b0'
46
+ # model_name: 'efficientnet_b3'
47
+ # pretrained: False
48
+ # frozen: False
49
+
50
+ ##### from scratch #####
51
+ # model_name: 'resnet34.a1_in1k'
52
+ # model_name: 'vit_base_patch16_clip_224.openai'
53
+ model_name: 'vit_large_patch14_clip_224' # Changed to match checkpoint cup_wild_vit_l_1img.ckpt (ViT-L with CLIP weights)
54
+ # model_name: 'convnext_base.clip_laion2b_augreg_ft_in12k'
55
+ pretrained: True
56
+ frozen: False
57
+
58
+
59
+ global_pool: ''
60
+
61
+ # 'avg' or 'max' or 'soft_attention' or 'spatial_embedding' or 'transformer' or 'attention_pool_2d'
62
+ feature_aggregation: 'attention_pool_2d'
63
+ # feature_aggregation: null
64
+ position_encording: 'sinusoidal' # 'learnable' or 'sinusoidal'. it only works for transformer
65
+
66
+ # it only works for resnet. 32 (7x7) or 16 (14x14)
67
+ downsample_ratio: 32
68
+
69
+ transforms:
70
+ - type: RandomCrop
71
+ ratio: 0.95
72
+ # RandomRotation is not compatible with BF16
73
+ # - _target_: torchvision.transforms.RandomRotation
74
+ # degrees:
75
+ # - -5.0
76
+ # - 5.0
77
+ # expand: false
78
+ - _target_: torchvision.transforms.ColorJitter
79
+ brightness: 0.3
80
+ contrast: 0.4
81
+ saturation: 0.5
82
+ hue: 0.08
83
+ # - _target_: torchvision.transforms.RandomPerspective
84
+ # distortion_scale: 0.5
85
+ # p: 1.0
86
+ # - _target_: torchvision.transforms.ElasticTransform
87
+ # alpha: 50.0
88
+ # sigma: 5.0
89
+ # - _target_: torchvision.transforms.RandomPerspective
90
+ # distortion_scale: 0.5
91
+ # p: 1.0
92
+ # - _target_: torchvision.transforms.ElasticTransform
93
+ # alpha: 50.0
94
+ # sigma: 5.0
95
+ # - _target_: torchvision.transforms.RandomCrop
96
+ # size: 192
97
+
98
+
99
+ use_group_norm: True
100
+ share_rgb_model: False
101
+ imagenet_norm: True
102
+
103
+ num_inference_steps: 16
104
+ obs_as_global_cond: True
105
+ diffusion_step_embed_dim: 128
106
+ down_dims: [256,512,1024]
107
+ kernel_size: 5
108
+ n_groups: 8
109
+ cond_predict_scale: True
110
+ input_pertub: 0.1 # reference: https://github.com/forever208/DDPM-IP
111
+ # train multiple diffusion steps per obs encoder forward pass
112
+ # doing this too much will cause instability!
113
+ train_diffusion_n_samples: 1
114
+
115
+ ema:
116
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
117
+ update_after_step: 0
118
+ inv_gamma: 1.0
119
+ power: 0.75
120
+ min_value: 0.0
121
+ max_value: 0.9999
122
+
123
+ dataloader:
124
+ batch_size: 64
125
+ num_workers: 8
126
+ shuffle: True
127
+ pin_memory: True
128
+ persistent_workers: True
129
+
130
+ val_dataloader:
131
+ batch_size: 64
132
+ num_workers: 8
133
+ shuffle: False
134
+ pin_memory: True
135
+ persistent_workers: True
136
+
137
+ # dataloader:
138
+ # batch_size: 64
139
+ # num_workers: 16
140
+ # shuffle: True
141
+ # pin_memory: True
142
+ # persistent_workers: True
143
+
144
+ # val_dataloader:
145
+ # batch_size: 64
146
+ # num_workers: 16
147
+ # shuffle: False
148
+ # pin_memory: True
149
+ # persistent_workers: True
150
+
151
+ optimizer:
152
+ _target_: torch.optim.AdamW
153
+ lr: 3.0e-4
154
+ betas: [0.95, 0.999]
155
+ eps: 1.0e-8
156
+ weight_decay: 1.0e-6
157
+
158
+ training:
159
+ device: "cuda:0"
160
+ seed: 42
161
+ debug: False
162
+ resume: False
163
+ # optimization
164
+ lr_scheduler: cosine
165
+ lr_warmup_steps: 2000
166
+ num_epochs: 120
167
+ gradient_accumulate_every: 1
168
+ # EMA destroys performance when used with BatchNorm
169
+ # replace BatchNorm with GroupNorm.
170
+ use_ema: True
171
+ freeze_encoder: False
172
+ # training loop control
173
+ # in epochs
174
+ rollout_every: 10
175
+ checkpoint_every: 10
176
+ val_every: 1
177
+ sample_every: 5
178
+ # steps per epoch
179
+ max_train_steps: null
180
+ max_val_steps: null
181
+ # misc
182
+ tqdm_interval_sec: 1.0
183
+
184
+ logging:
185
+ project: umi
186
+ resume: False
187
+ mode: online
188
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
189
+ tags: ["${name}", "${task_name}", "${exp_name}"]
190
+ id: null
191
+ group: null
192
+
193
+ checkpoint:
194
+ topk:
195
+ monitor_key: train_loss
196
+ mode: min
197
+ k: 20
198
+ format_str: 'epoch_{epoch:04d}-train_loss_{train_loss:.3f}.ckpt'
199
+ save_last_ckpt: True
200
+ save_last_snapshot: False
201
+
202
+ multi_run:
203
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
204
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
205
+
206
+ hydra:
207
+ job:
208
+ override_dirname: ${name}
209
+ run:
210
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
211
+ sweep:
212
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
213
+ subdir: ${hydra.job.num}