solarian-diffusion 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. diffusion_policy/__init__.py +0 -0
  2. diffusion_policy/codecs/__init__.py +0 -0
  3. diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
  4. diffusion_policy/common/__init__.py +0 -0
  5. diffusion_policy/common/checkpoint_util.py +64 -0
  6. diffusion_policy/common/json_logger.py +117 -0
  7. diffusion_policy/common/normalize_util.py +251 -0
  8. diffusion_policy/common/pose_repr_util.py +112 -0
  9. diffusion_policy/common/pytorch_util.py +82 -0
  10. diffusion_policy/common/q_critic_util.py +135 -0
  11. diffusion_policy/common/replay_buffer.py +672 -0
  12. diffusion_policy/common/sampler.py +204 -0
  13. diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
  14. diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
  15. diffusion_policy/config/legacy/task/can_image.yaml +64 -0
  16. diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
  17. diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
  18. diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
  19. diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
  20. diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
  21. diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
  22. diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
  23. diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
  24. diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
  25. diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
  26. diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
  27. diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
  28. diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
  29. diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
  30. diffusion_policy/config/legacy/task/square_image.yaml +64 -0
  31. diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
  32. diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
  33. diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
  34. diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
  35. diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
  36. diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
  37. diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
  38. diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
  39. diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
  40. diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
  41. diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
  42. diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
  43. diffusion_policy/config/legacy/task/umi.yaml +45 -0
  44. diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
  45. diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
  46. diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
  47. diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
  48. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
  49. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
  50. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
  51. diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
  52. diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
  53. diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
  54. diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
  55. diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
  56. diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
  57. diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
  58. diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
  59. diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
  60. diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
  61. diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
  62. diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
  63. diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
  64. diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
  65. diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
  66. diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
  67. diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
  68. diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
  69. diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
  70. diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
  71. diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
  72. diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
  73. diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
  74. diffusion_policy/config/rl_annotate.yaml +265 -0
  75. diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
  76. diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
  77. diffusion_policy/config/rl_annotate_v2.yaml +141 -0
  78. diffusion_policy/config/rl_annotate_v3.yaml +165 -0
  79. diffusion_policy/config/task/lift.yaml +89 -0
  80. diffusion_policy/config/task/square.yaml +92 -0
  81. diffusion_policy/config/task/tool_hang.yaml +93 -0
  82. diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
  83. diffusion_policy/config/task/umi.yaml +105 -0
  84. diffusion_policy/config/task/umi_bimanual.yaml +135 -0
  85. diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
  86. diffusion_policy/config/task/umi_image.yaml +36 -0
  87. diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
  88. diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
  89. diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
  90. diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
  91. diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
  92. diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
  93. diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
  94. diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
  95. diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
  96. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
  97. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
  98. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
  99. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
  100. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
  101. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
  102. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
  103. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
  104. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
  105. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
  106. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
  107. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
  108. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
  109. diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
  110. diffusion_policy/config/train_iql_umi.yaml +186 -0
  111. diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
  112. diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
  113. diffusion_policy/dataset/__init__.py +0 -0
  114. diffusion_policy/dataset/base_dataset.py +76 -0
  115. diffusion_policy/dataset/umi_dataset.py +652 -0
  116. diffusion_policy/env_runner/__init__.py +0 -0
  117. diffusion_policy/env_runner/base_image_runner.py +9 -0
  118. diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
  119. diffusion_policy/model/__init__.py +0 -0
  120. diffusion_policy/model/common/__init__.py +0 -0
  121. diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
  122. diffusion_policy/model/common/lr_scheduler.py +46 -0
  123. diffusion_policy/model/common/module_attr_mixin.py +14 -0
  124. diffusion_policy/model/common/normalizer.py +353 -0
  125. diffusion_policy/model/common/tensor_util.py +960 -0
  126. diffusion_policy/model/diffusion/__init__.py +0 -0
  127. diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
  128. diffusion_policy/model/diffusion/conv1d_components.py +46 -0
  129. diffusion_policy/model/diffusion/ema_model.py +88 -0
  130. diffusion_policy/model/diffusion/mask_generator.py +221 -0
  131. diffusion_policy/model/diffusion/positional_embedding.py +17 -0
  132. diffusion_policy/model/vision/__init__.py +0 -0
  133. diffusion_policy/model/vision/crop_randomizer.py +288 -0
  134. diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
  135. diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
  136. diffusion_policy/policy/__init__.py +0 -0
  137. diffusion_policy/policy/base_image_policy.py +27 -0
  138. diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
  139. diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
  140. diffusion_policy/policy/parl_action_optimizer.py +675 -0
  141. diffusion_policy/workspace/__init__.py +0 -0
  142. diffusion_policy/workspace/base_workspace.py +146 -0
  143. diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
  144. solarian_diffusion/__init__.py +32 -0
  145. solarian_diffusion/config.py +31 -0
  146. solarian_diffusion/policy_server.py +316 -0
  147. solarian_diffusion/rtc.py +272 -0
  148. solarian_diffusion/service.py +16 -0
  149. solarian_diffusion/trainer.py +213 -0
  150. solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
  151. solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
  152. solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
  153. solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
  154. solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
  155. solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
  156. solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
  157. umi/__init__.py +0 -0
  158. umi/common/__init__.py +0 -0
  159. umi/common/pose_util.py +115 -0
@@ -0,0 +1,152 @@
1
+ defaults:
2
+ - _self_
3
+ - task: lift_image_abs
4
+
5
+ name: train_diffusion_unet_timm
6
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
7
+
8
+ task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+
12
+ n_action_steps: 8
13
+
14
+ policy:
15
+ _target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
16
+
17
+ shape_meta: ${shape_meta}
18
+
19
+ noise_scheduler:
20
+ _target_: diffusers.DDIMScheduler
21
+ num_train_timesteps: 50
22
+ beta_start: 0.0001
23
+ beta_end: 0.02
24
+ # beta_schedule is important
25
+ # this is the best we found
26
+ beta_schedule: squaredcos_cap_v2
27
+ clip_sample: True
28
+ set_alpha_to_one: True
29
+ steps_offset: 0
30
+ prediction_type: epsilon # or sample
31
+
32
+ obs_encoder:
33
+ _target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
34
+ shape_meta: ${shape_meta}
35
+ model_name: 'resnet18'
36
+ # model_name: 'vit_base_patch16_224'
37
+ pretrained: False
38
+ global_pool: 'avg'
39
+ transforms:
40
+ # - _target_: torchvision.transforms.Resize
41
+ # size: 224
42
+ # antialias: True
43
+ - _target_: torchvision.transforms.RandomRotation
44
+ degrees:
45
+ - -5.0
46
+ - 5.0
47
+ expand: false
48
+ - _target_: torchvision.transforms.ColorJitter
49
+ brightness: 0.2
50
+ contrast: 0.5
51
+ saturation: 0.2
52
+ hue: 0.2
53
+
54
+ use_group_norm: True
55
+ share_rgb_model: False
56
+ imagenet_norm: True
57
+
58
+ num_inference_steps: 16
59
+ obs_as_global_cond: True
60
+ diffusion_step_embed_dim: 128
61
+ down_dims: [256,512,1024]
62
+ kernel_size: 5
63
+ n_groups: 8
64
+ cond_predict_scale: True
65
+ input_pertub: 0.1 # reference: https://github.com/forever208/DDPM-IP
66
+ inpaint_fixed_action_prefix: False
67
+
68
+ ema:
69
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
70
+ update_after_step: 0
71
+ inv_gamma: 1.0
72
+ power: 0.75
73
+ min_value: 0.0
74
+ max_value: 0.9999
75
+
76
+ dataloader:
77
+ batch_size: ${eval:'128 if "${task.name}" == "tool_hang" else 256'}
78
+ num_workers: 16
79
+ shuffle: True
80
+ pin_memory: True
81
+ persistent_workers: True
82
+
83
+ val_dataloader:
84
+ batch_size: ${eval:'128 if "${task.name}" == "tool_hang" else 256'}
85
+ num_workers: 16
86
+ shuffle: False
87
+ pin_memory: True
88
+ persistent_workers: True
89
+
90
+ optimizer:
91
+ _target_: torch.optim.AdamW
92
+ lr: 1.0e-4
93
+ betas: [0.95, 0.999]
94
+ eps: 1.0e-8
95
+ weight_decay: 1.0e-6
96
+
97
+ training:
98
+ device: "cuda:0"
99
+ seed: 42
100
+ debug: False
101
+ resume: True
102
+ # optimization
103
+ lr_scheduler: cosine
104
+ lr_warmup_steps: 500
105
+ num_epochs: 600
106
+ gradient_accumulate_every: 1
107
+ # EMA destroys performance when used with BatchNorm
108
+ # replace BatchNorm with GroupNorm.
109
+ use_ema: True
110
+ freeze_encoder: False
111
+ # training loop control
112
+ # in epochs
113
+ rollout_every: 50
114
+ checkpoint_every: 50
115
+ val_every: 1
116
+ sample_every: 5
117
+ # steps per epoch
118
+ max_train_steps: null
119
+ max_val_steps: null
120
+ # misc
121
+ tqdm_interval_sec: 1.0
122
+
123
+ logging:
124
+ project: diffusion_policy_debug
125
+ resume: True
126
+ mode: online
127
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
128
+ tags: ["${name}", "${task_name}", "${exp_name}"]
129
+ id: null
130
+ group: null
131
+
132
+ checkpoint:
133
+ topk:
134
+ monitor_key: test_mean_score
135
+ mode: max
136
+ k: 5
137
+ format_str: 'epoch_{epoch:04d}-test_mean_score_{test_mean_score:.3f}.ckpt'
138
+ save_last_ckpt: True
139
+ save_last_snapshot: False
140
+
141
+ multi_run:
142
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
143
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
144
+
145
+ hydra:
146
+ job:
147
+ override_dirname: ${name}
148
+ run:
149
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
150
+ sweep:
151
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
152
+ subdir: ${hydra.job.num}
@@ -0,0 +1,260 @@
1
+ defaults:
2
+ - _self_
3
+ - task: umi_cup_wild
4
+
5
+ name: train_diffusion_unet_timm
6
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
7
+
8
+ # task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+ batch_size: 64
12
+ # visual_encoder: "vit_base_patch16_clip_224.openai"
13
+ # visual_encoder_tag: "vitb"
14
+ visual_encoder: "vit_large_patch14_clip_224"
15
+ visual_encoder_tag: "vitl"
16
+ # visual_encoder: "resnet34.a1_in1k"
17
+ # visual_encoder_tag: "resnet34"
18
+ model_type: "unet"
19
+ # this will be overridden at call time
20
+ num_gpus: 1
21
+
22
+
23
+ # Parsed from the dataset file name's hyphen-separated tokens.
24
+ # eg: pick_lego-black_mirror-real_200-sim_200-no_bbox.zarr.zip
25
+ # Resolver: train.py::parse_dataset_field. Last arg is the fallback default.
26
+ # mirror options: black_mirror, mirror, swap_mirror, no_mirror
27
+ mirror: ${parse_dataset_field:${task.dataset_path},mirror,no_mirror}
28
+ # real_data_size / sim_data_size are null when the filename has no real_<N>/sim_<N>
29
+ # token; the wandb tags below skip themselves in that case.
30
+ real_data_size: ${parse_dataset_field:${task.dataset_path},real_data_size,null}
31
+ sim_data_size: ${parse_dataset_field:${task.dataset_path},sim_data_size,null}
32
+ bbox: ${parse_dataset_field:${task.dataset_path},bbox,no_bbox}
33
+ task_name: ${parse_dataset_field:${task.dataset_path},data_task,pick_lego}
34
+
35
+ # Override dataset path for fine-tuning
36
+ # This will override the dataset_path in task/umi.yaml
37
+ # Uncomment and set to your new dataset path:
38
+ # task.dataset_path: "path/to/your/new/dataset.zarr.zip"
39
+
40
+ n_action_steps: 8
41
+
42
+ policy:
43
+ _target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
44
+
45
+ shape_meta: ${shape_meta}
46
+
47
+ noise_scheduler:
48
+ _target_: diffusers.DDIMScheduler
49
+ num_train_timesteps: 50
50
+ beta_start: 0.0001
51
+ beta_end: 0.02
52
+ # beta_schedule is important
53
+ # this is the best we found
54
+ beta_schedule: squaredcos_cap_v2
55
+ clip_sample: True
56
+ set_alpha_to_one: True
57
+ steps_offset: 0
58
+ prediction_type: epsilon # or sample
59
+
60
+ obs_encoder:
61
+ _target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
62
+ shape_meta: ${shape_meta}
63
+
64
+ ##### from scratch #####
65
+ # model_name: 'vit_base_patch16_224'
66
+ # model_name: 'resnet34'
67
+ # model_name: 'vit_tiny_patch16_224'
68
+ # model_name: 'efficientnet_b0'
69
+ # model_name: 'efficientnet_b3'
70
+ # pretrained: False
71
+ # frozen: False
72
+
73
+ ##### from scratch #####
74
+ # model_name: 'resnet34.a1_in1k'
75
+ model_name: ${visual_encoder}
76
+ # model_name: 'vit_large_patch14_clip_224' # Changed to match checkpoint cup_wild_vit_l_1img.ckpt (ViT-L with CLIP weights)
77
+ # model_name: 'convnext_base.clip_laion2b_augreg_ft_in12k'
78
+ pretrained: True
79
+ frozen: False
80
+
81
+
82
+ global_pool: ''
83
+
84
+ # 'avg' or 'max' or 'soft_attention' or 'spatial_embedding' or 'transformer' or 'attention_pool_2d'
85
+ feature_aggregation: 'attention_pool_2d'
86
+ # feature_aggregation: null
87
+ position_encording: 'sinusoidal' # 'learnable' or 'sinusoidal'. it only works for transformer
88
+
89
+ # it only works for resnet. 32 (7x7) or 16 (14x14)
90
+ downsample_ratio: 32
91
+
92
+ transforms:
93
+ - type: RandomCrop
94
+ ratio: 0.95
95
+ # RandomRotation is not compatible with BF16
96
+ # - _target_: torchvision.transforms.RandomRotation
97
+ # degrees:
98
+ # - -5.0
99
+ # - 5.0
100
+ # expand: false
101
+ - _target_: torchvision.transforms.ColorJitter
102
+ brightness: 0.3
103
+ contrast: 0.4
104
+ saturation: 0.5
105
+ hue: 0.08
106
+ # - _target_: torchvision.transforms.RandomPerspective
107
+ # distortion_scale: 0.5
108
+ # p: 1.0
109
+ # - _target_: torchvision.transforms.ElasticTransform
110
+ # alpha: 50.0
111
+ # sigma: 5.0
112
+ # - _target_: torchvision.transforms.RandomPerspective
113
+ # distortion_scale: 0.5
114
+ # p: 1.0
115
+ # - _target_: torchvision.transforms.ElasticTransform
116
+ # alpha: 50.0
117
+ # sigma: 5.0
118
+ # - _target_: torchvision.transforms.RandomCrop
119
+ # size: 192
120
+
121
+
122
+ use_group_norm: True
123
+ share_rgb_model: False
124
+ imagenet_norm: True
125
+
126
+ num_inference_steps: 16
127
+ obs_as_global_cond: True
128
+ diffusion_step_embed_dim: 128
129
+ down_dims: [256,512,1024]
130
+ kernel_size: 5
131
+ n_groups: 8
132
+ cond_predict_scale: True
133
+ input_pertub: 0.1 # reference: https://github.com/forever208/DDPM-IP
134
+ # train multiple diffusion steps per obs encoder forward pass
135
+ # doing this too much will cause instability!
136
+ train_diffusion_n_samples: 1
137
+
138
+ ema:
139
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
140
+ update_after_step: 0
141
+ inv_gamma: 1.0
142
+ power: 0.75
143
+ min_value: 0.0
144
+ max_value: 0.9999
145
+
146
+ dataloader:
147
+ batch_size: ${batch_size}
148
+ num_workers: 8
149
+ shuffle: True
150
+ pin_memory: True
151
+ persistent_workers: True
152
+ prefetch_factor: 4
153
+
154
+ val_dataloader:
155
+ batch_size: ${batch_size}
156
+ num_workers: 8
157
+ shuffle: False
158
+ pin_memory: True
159
+ persistent_workers: True
160
+ prefetch_factor: 4
161
+
162
+ # dataloader:
163
+ # batch_size: 64
164
+ # num_workers: 16
165
+ # shuffle: True
166
+ # pin_memory: True
167
+ # persistent_workers: True
168
+
169
+ # val_dataloader:
170
+ # batch_size: 64
171
+ # num_workers: 16
172
+ # shuffle: False
173
+ # pin_memory: True
174
+ # persistent_workers: True
175
+
176
+ optimizer:
177
+ _target_: torch.optim.AdamW
178
+ lr: 3.0e-4
179
+ betas: [0.95, 0.999]
180
+ eps: 1.0e-8
181
+ weight_decay: 1.0e-6
182
+
183
+ training:
184
+ device: "cuda:0"
185
+ seed: 42
186
+ debug: False
187
+ resume: False
188
+ # optimization
189
+ lr_scheduler: cosine
190
+ lr_warmup_steps: 500
191
+ num_epochs: 50
192
+ gradient_accumulate_every: 1
193
+ # EMA destroys performance when used with BatchNorm
194
+ # replace BatchNorm with GroupNorm.
195
+ use_ema: True
196
+ freeze_encoder: False
197
+ # training loop control
198
+ # in epochs
199
+ rollout_every: 10
200
+ checkpoint_every: 10
201
+ val_every: 1
202
+ sample_every: 5
203
+ # steps per epoch
204
+ max_train_steps: null
205
+ max_val_steps: null
206
+ # misc
207
+ tqdm_interval_sec: 1.0
208
+
209
+ logging:
210
+ project: umi
211
+ resume: False
212
+ mode: online
213
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
214
+ tags:
215
+ - "${name}"
216
+ - "${task_name}"
217
+ - "${exp_name}"
218
+ # visual encoder could be vit_base_patch16_clip_224.openai, resnet34, etc.
219
+ - "${visual_encoder_tag}"
220
+ # batch size for each gpu
221
+ - "batch_size:${eval:'${batch_size} * ${num_gpus}'}"
222
+ - "num_gpus:${num_gpus}"
223
+ # model type could be unet or transformer
224
+ - "${model_type}"
225
+ # dataset file name (joined; substring-searchable on wandb)
226
+ # Note: full ${task.dataset_path} dropped — wandb tags max 64 chars.
227
+ - "${eval:'\"${task.dataset_path}\".split(\"/\")[-1].rsplit(\".\",1)[0]'}"
228
+ - "${mirror}"
229
+ # Only emit real_/sim_ tags when the dataset filename actually carries
230
+ # a real_<N> / sim_<N> token (otherwise the resolver returns None and we
231
+ # render an empty string here, which is filtered out before wandb init).
232
+ - "${eval:'\"real_size:%d\" % ${real_data_size} if ${real_data_size} is not None else \"\"'}"
233
+ - "${eval:'\"sim_size:%d\" % ${sim_data_size} if ${sim_data_size} is not None else \"\"'}"
234
+ - "${eval:'(\"real\" if ${real_data_size} > 0 else \"no_real\") if ${real_data_size} is not None else \"\"'}"
235
+ - "${eval:'(\"sim\" if ${sim_data_size} > 0 else \"no_sim\") if ${sim_data_size} is not None else \"\"'}"
236
+ - "${bbox}"
237
+ id: null
238
+ group: ${task_name}_${model_type}
239
+
240
+ checkpoint:
241
+ topk:
242
+ monitor_key: train_loss
243
+ mode: min
244
+ k: 20
245
+ format_str: '{task_name}-{model_type}-{visual_encoder}-batch_{total_batch}-{dataset_file_name}-epoch_{epoch:04d}-train_loss_{train_loss:.3f}.ckpt'
246
+ save_last_ckpt: True
247
+ save_last_snapshot: False
248
+
249
+ multi_run:
250
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
251
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
252
+
253
+ hydra:
254
+ job:
255
+ override_dirname: ${name}
256
+ run:
257
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
258
+ sweep:
259
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
260
+ subdir: ${hydra.job.num}
@@ -0,0 +1,265 @@
1
+ defaults:
2
+ - _self_
3
+ - task: umi_pick_single_lego_bbox
4
+
5
+ name: train_diffusion_unet_timm
6
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
7
+
8
+ # task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+ batch_size: 128
12
+ # visual_encoder: "vit_base_patch16_clip_224.openai"
13
+ # visual_encoder_tag: "vitb"
14
+ visual_encoder: "vit_large_patch14_clip_224"
15
+ visual_encoder_tag: "vitl"
16
+ # visual_encoder: "resnet34.a1_in1k"
17
+ # visual_encoder_tag: "resnet34"
18
+ model_type: "unet"
19
+ # this will be overridden at call time
20
+ num_gpus: 1
21
+
22
+
23
+ # Parsed from the dataset file name's hyphen-separated tokens.
24
+ # eg: pick_lego-black_mirror-real_200-sim_200-no_bbox.zarr.zip
25
+ # Resolver: train.py::parse_dataset_field. Last arg is the fallback default.
26
+ # mirror options: black_mirror, mirror, swap_mirror, no_mirror
27
+ mirror: ${parse_dataset_field:${task.dataset_path},mirror,no_mirror}
28
+ # real_data_size / sim_data_size are null when the filename has no real_<N>/sim_<N>
29
+ # token; the wandb tags below skip themselves in that case.
30
+ real_data_size: ${parse_dataset_field:${task.dataset_path},real_data_size,null}
31
+ sim_data_size: ${parse_dataset_field:${task.dataset_path},sim_data_size,null}
32
+ bbox: ${parse_dataset_field:${task.dataset_path},bbox,no_bbox}
33
+ # Surfaces the bbox integration mode (low_dim vs mask_channel) onto the
34
+ # wandb tag set so runs are filterable by it. Sourced from the task
35
+ # shape_meta so it tracks the actual model wiring, not a separate flag.
36
+ bbox_mode: ${task.shape_meta.obs.bbox.bbox_mode}
37
+ task_name: ${parse_dataset_field:${task.dataset_path},data_task,pick_lego}
38
+
39
+ # Override dataset path for fine-tuning
40
+ # This will override the dataset_path in task/umi.yaml
41
+ # Uncomment and set to your new dataset path:
42
+ # task.dataset_path: "path/to/your/new/dataset.zarr.zip"
43
+
44
+ n_action_steps: 8
45
+
46
+ policy:
47
+ _target_: diffusion_policy.policy.diffusion_unet_timm_policy.DiffusionUnetTimmPolicy
48
+
49
+ shape_meta: ${shape_meta}
50
+
51
+ noise_scheduler:
52
+ _target_: diffusers.DDIMScheduler
53
+ num_train_timesteps: 50
54
+ beta_start: 0.0001
55
+ beta_end: 0.02
56
+ # beta_schedule is important
57
+ # this is the best we found
58
+ beta_schedule: squaredcos_cap_v2
59
+ clip_sample: True
60
+ set_alpha_to_one: True
61
+ steps_offset: 0
62
+ prediction_type: epsilon # or sample
63
+
64
+ obs_encoder:
65
+ _target_: diffusion_policy.model.vision.timm_obs_encoder.TimmObsEncoder
66
+ shape_meta: ${shape_meta}
67
+
68
+ ##### from scratch #####
69
+ # model_name: 'vit_base_patch16_224'
70
+ # model_name: 'resnet34'
71
+ # model_name: 'vit_tiny_patch16_224'
72
+ # model_name: 'efficientnet_b0'
73
+ # model_name: 'efficientnet_b3'
74
+ # pretrained: False
75
+ # frozen: False
76
+
77
+ ##### from scratch #####
78
+ # model_name: 'resnet34.a1_in1k'
79
+ model_name: ${visual_encoder}
80
+ # model_name: 'vit_large_patch14_clip_224' # Changed to match checkpoint cup_wild_vit_l_1img.ckpt (ViT-L with CLIP weights)
81
+ # model_name: 'convnext_base.clip_laion2b_augreg_ft_in12k'
82
+ pretrained: True
83
+ frozen: False
84
+
85
+
86
+ global_pool: ''
87
+
88
+ # 'avg' or 'max' or 'soft_attention' or 'spatial_embedding' or 'transformer' or 'attention_pool_2d'
89
+ feature_aggregation: 'attention_pool_2d'
90
+ # feature_aggregation: null
91
+ position_encording: 'sinusoidal' # 'learnable' or 'sinusoidal'. it only works for transformer
92
+
93
+ # it only works for resnet. 32 (7x7) or 16 (14x14)
94
+ downsample_ratio: 32
95
+
96
+ transforms:
97
+ - type: RandomCrop
98
+ ratio: 0.95
99
+ # RandomRotation is not compatible with BF16
100
+ # - _target_: torchvision.transforms.RandomRotation
101
+ # degrees:
102
+ # - -5.0
103
+ # - 5.0
104
+ # expand: false
105
+ - _target_: torchvision.transforms.ColorJitter
106
+ brightness: 0.3
107
+ contrast: 0.4
108
+ saturation: 0.5
109
+ hue: 0.08
110
+ # - _target_: torchvision.transforms.RandomPerspective
111
+ # distortion_scale: 0.5
112
+ # p: 1.0
113
+ # - _target_: torchvision.transforms.ElasticTransform
114
+ # alpha: 50.0
115
+ # sigma: 5.0
116
+ # - _target_: torchvision.transforms.RandomPerspective
117
+ # distortion_scale: 0.5
118
+ # p: 1.0
119
+ # - _target_: torchvision.transforms.ElasticTransform
120
+ # alpha: 50.0
121
+ # sigma: 5.0
122
+ # - _target_: torchvision.transforms.RandomCrop
123
+ # size: 192
124
+
125
+
126
+ use_group_norm: True
127
+ share_rgb_model: False
128
+ imagenet_norm: True
129
+
130
+ num_inference_steps: 16
131
+ obs_as_global_cond: True
132
+ diffusion_step_embed_dim: 128
133
+ down_dims: [256,512,1024]
134
+ kernel_size: 5
135
+ n_groups: 8
136
+ cond_predict_scale: True
137
+ input_pertub: 0.1 # reference: https://github.com/forever208/DDPM-IP
138
+ # train multiple diffusion steps per obs encoder forward pass
139
+ # doing this too much will cause instability!
140
+ train_diffusion_n_samples: 1
141
+
142
+ ema:
143
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
144
+ update_after_step: 0
145
+ inv_gamma: 1.0
146
+ power: 0.75
147
+ min_value: 0.0
148
+ max_value: 0.9999
149
+
150
+ dataloader:
151
+ batch_size: ${batch_size}
152
+ num_workers: 8
153
+ shuffle: True
154
+ pin_memory: True
155
+ persistent_workers: True
156
+ prefetch_factor: 4
157
+
158
+ val_dataloader:
159
+ batch_size: ${batch_size}
160
+ num_workers: 8
161
+ shuffle: False
162
+ pin_memory: True
163
+ persistent_workers: True
164
+ prefetch_factor: 4
165
+
166
+ # dataloader:
167
+ # batch_size: 64
168
+ # num_workers: 16
169
+ # shuffle: True
170
+ # pin_memory: True
171
+ # persistent_workers: True
172
+
173
+ # val_dataloader:
174
+ # batch_size: 64
175
+ # num_workers: 16
176
+ # shuffle: False
177
+ # pin_memory: True
178
+ # persistent_workers: True
179
+
180
+ optimizer:
181
+ _target_: torch.optim.AdamW
182
+ lr: 3.0e-4
183
+ betas: [0.95, 0.999]
184
+ eps: 1.0e-8
185
+ weight_decay: 1.0e-6
186
+
187
+ training:
188
+ device: "cuda:0"
189
+ seed: 42
190
+ debug: False
191
+ resume: False
192
+ # optimization
193
+ lr_scheduler: cosine
194
+ lr_warmup_steps: 500
195
+ num_epochs: 200
196
+ gradient_accumulate_every: 1
197
+ # EMA destroys performance when used with BatchNorm
198
+ # replace BatchNorm with GroupNorm.
199
+ use_ema: True
200
+ freeze_encoder: False
201
+ # training loop control
202
+ # in epochs
203
+ rollout_every: 10
204
+ checkpoint_every: 10
205
+ val_every: 1
206
+ sample_every: 5
207
+ # steps per epoch
208
+ max_train_steps: null
209
+ max_val_steps: null
210
+ # misc
211
+ tqdm_interval_sec: 1.0
212
+
213
+ logging:
214
+ project: umi
215
+ resume: False
216
+ mode: online
217
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
218
+ tags:
219
+ - "${name}"
220
+ - "${task_name}"
221
+ - "${exp_name}"
222
+ # visual encoder could be vit_base_patch16_clip_224.openai, resnet34, etc.
223
+ - "${visual_encoder_tag}"
224
+ # batch size for each gpu
225
+ - "batch_size:${eval:'${batch_size} * ${num_gpus}'}"
226
+ - "num_gpus:${num_gpus}"
227
+ # model type could be unet or transformer
228
+ - "${model_type}"
229
+ # dataset file name (joined; substring-searchable on wandb)
230
+ # Note: full ${task.dataset_path} dropped — wandb tags max 64 chars.
231
+ - "${eval:'\"${task.dataset_path}\".split(\"/\")[-1].rsplit(\".\",1)[0]'}"
232
+ - "${mirror}"
233
+ # Only emit real_/sim_ tags when the dataset filename actually carries
234
+ # a real_<N> / sim_<N> token (otherwise the resolver returns None and we
235
+ # render an empty string here, which is filtered out before wandb init).
236
+ - "${eval:'\"real_size:%d\" % ${real_data_size} if ${real_data_size} is not None else \"\"'}"
237
+ - "${eval:'\"sim_size:%d\" % ${sim_data_size} if ${sim_data_size} is not None else \"\"'}"
238
+ - "${eval:'(\"real\" if ${real_data_size} > 0 else \"no_real\") if ${real_data_size} is not None else \"\"'}"
239
+ - "${eval:'(\"sim\" if ${sim_data_size} > 0 else \"no_sim\") if ${sim_data_size} is not None else \"\"'}"
240
+ - "${bbox}"
241
+ - "bbox_mode:${bbox_mode}"
242
+ id: null
243
+ group: ${task_name}_${model_type}
244
+
245
+ checkpoint:
246
+ topk:
247
+ monitor_key: train_loss
248
+ mode: min
249
+ k: 20
250
+ format_str: '{task_name}-{model_type}-{visual_encoder}-batch_{total_batch}-{dataset_file_name}-epoch_{epoch:04d}-train_loss_{train_loss:.3f}.ckpt'
251
+ save_last_ckpt: True
252
+ save_last_snapshot: False
253
+
254
+ multi_run:
255
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
256
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
257
+
258
+ hydra:
259
+ job:
260
+ override_dirname: ${name}
261
+ run:
262
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
263
+ sweep:
264
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
265
+ subdir: ${hydra.job.num}