solarian-diffusion 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. diffusion_policy/__init__.py +0 -0
  2. diffusion_policy/codecs/__init__.py +0 -0
  3. diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
  4. diffusion_policy/common/__init__.py +0 -0
  5. diffusion_policy/common/checkpoint_util.py +64 -0
  6. diffusion_policy/common/json_logger.py +117 -0
  7. diffusion_policy/common/normalize_util.py +251 -0
  8. diffusion_policy/common/pose_repr_util.py +112 -0
  9. diffusion_policy/common/pytorch_util.py +82 -0
  10. diffusion_policy/common/q_critic_util.py +135 -0
  11. diffusion_policy/common/replay_buffer.py +672 -0
  12. diffusion_policy/common/sampler.py +204 -0
  13. diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
  14. diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
  15. diffusion_policy/config/legacy/task/can_image.yaml +64 -0
  16. diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
  17. diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
  18. diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
  19. diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
  20. diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
  21. diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
  22. diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
  23. diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
  24. diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
  25. diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
  26. diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
  27. diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
  28. diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
  29. diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
  30. diffusion_policy/config/legacy/task/square_image.yaml +64 -0
  31. diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
  32. diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
  33. diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
  34. diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
  35. diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
  36. diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
  37. diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
  38. diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
  39. diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
  40. diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
  41. diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
  42. diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
  43. diffusion_policy/config/legacy/task/umi.yaml +45 -0
  44. diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
  45. diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
  46. diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
  47. diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
  48. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
  49. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
  50. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
  51. diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
  52. diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
  53. diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
  54. diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
  55. diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
  56. diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
  57. diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
  58. diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
  59. diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
  60. diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
  61. diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
  62. diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
  63. diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
  64. diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
  65. diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
  66. diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
  67. diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
  68. diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
  69. diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
  70. diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
  71. diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
  72. diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
  73. diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
  74. diffusion_policy/config/rl_annotate.yaml +265 -0
  75. diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
  76. diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
  77. diffusion_policy/config/rl_annotate_v2.yaml +141 -0
  78. diffusion_policy/config/rl_annotate_v3.yaml +165 -0
  79. diffusion_policy/config/task/lift.yaml +89 -0
  80. diffusion_policy/config/task/square.yaml +92 -0
  81. diffusion_policy/config/task/tool_hang.yaml +93 -0
  82. diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
  83. diffusion_policy/config/task/umi.yaml +105 -0
  84. diffusion_policy/config/task/umi_bimanual.yaml +135 -0
  85. diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
  86. diffusion_policy/config/task/umi_image.yaml +36 -0
  87. diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
  88. diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
  89. diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
  90. diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
  91. diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
  92. diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
  93. diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
  94. diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
  95. diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
  96. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
  97. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
  98. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
  99. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
  100. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
  101. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
  102. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
  103. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
  104. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
  105. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
  106. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
  107. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
  108. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
  109. diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
  110. diffusion_policy/config/train_iql_umi.yaml +186 -0
  111. diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
  112. diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
  113. diffusion_policy/dataset/__init__.py +0 -0
  114. diffusion_policy/dataset/base_dataset.py +76 -0
  115. diffusion_policy/dataset/umi_dataset.py +652 -0
  116. diffusion_policy/env_runner/__init__.py +0 -0
  117. diffusion_policy/env_runner/base_image_runner.py +9 -0
  118. diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
  119. diffusion_policy/model/__init__.py +0 -0
  120. diffusion_policy/model/common/__init__.py +0 -0
  121. diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
  122. diffusion_policy/model/common/lr_scheduler.py +46 -0
  123. diffusion_policy/model/common/module_attr_mixin.py +14 -0
  124. diffusion_policy/model/common/normalizer.py +353 -0
  125. diffusion_policy/model/common/tensor_util.py +960 -0
  126. diffusion_policy/model/diffusion/__init__.py +0 -0
  127. diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
  128. diffusion_policy/model/diffusion/conv1d_components.py +46 -0
  129. diffusion_policy/model/diffusion/ema_model.py +88 -0
  130. diffusion_policy/model/diffusion/mask_generator.py +221 -0
  131. diffusion_policy/model/diffusion/positional_embedding.py +17 -0
  132. diffusion_policy/model/vision/__init__.py +0 -0
  133. diffusion_policy/model/vision/crop_randomizer.py +288 -0
  134. diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
  135. diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
  136. diffusion_policy/policy/__init__.py +0 -0
  137. diffusion_policy/policy/base_image_policy.py +27 -0
  138. diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
  139. diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
  140. diffusion_policy/policy/parl_action_optimizer.py +675 -0
  141. diffusion_policy/workspace/__init__.py +0 -0
  142. diffusion_policy/workspace/base_workspace.py +146 -0
  143. diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
  144. solarian_diffusion/__init__.py +32 -0
  145. solarian_diffusion/config.py +31 -0
  146. solarian_diffusion/policy_server.py +316 -0
  147. solarian_diffusion/rtc.py +272 -0
  148. solarian_diffusion/service.py +16 -0
  149. solarian_diffusion/trainer.py +213 -0
  150. solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
  151. solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
  152. solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
  153. solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
  154. solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
  155. solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
  156. solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
  157. umi/__init__.py +0 -0
  158. umi/common/__init__.py +0 -0
  159. umi/common/pose_util.py +115 -0
@@ -0,0 +1,135 @@
1
+ name: umi_pick_lego
2
+
3
+ camera_obs_latency: 0.125
4
+ robot_obs_latency: 0.0001
5
+ gripper_obs_latency: 0.02
6
+ dataset_frequeny: 0 #59.94
7
+ obs_down_sample_steps: 3 # 3, 1
8
+
9
+ low_dim_obs_horizon: 2
10
+ img_obs_horizon: 1
11
+ action_horizon: 16
12
+ ignore_proprioception: False
13
+
14
+ shape_meta: &shape_meta
15
+ # acceptable types: rgb, low_dim
16
+ obs:
17
+ camera0_rgb:
18
+ shape: [3, 224, 224]
19
+ horizon: ${task.img_obs_horizon} # int
20
+ latency_steps: 0 # float
21
+ down_sample_steps: ${task.obs_down_sample_steps} # int
22
+ type: rgb
23
+ ignore_by_policy: False
24
+ robot0_eef_pos:
25
+ shape: [3]
26
+ horizon: ${task.low_dim_obs_horizon} # int
27
+ latency_steps: ${eval:'(${task.camera_obs_latency} - ${task.robot_obs_latency}) * ${task.dataset_frequeny}'} # float
28
+ down_sample_steps: ${task.obs_down_sample_steps} # float
29
+ type: low_dim
30
+ ignore_by_policy: ${task.ignore_proprioception}
31
+ # robot0_eef_pos_abs:
32
+ # shape: [1]
33
+ # axis: [2] # 2 means z-axis
34
+ # horizon: ${task.low_dim_obs_horizon} # int
35
+ # latency_steps: ${eval:'(${task.camera_obs_latency} - ${task.robot_obs_latency}) * ${task.dataset_frequeny}'} # float
36
+ # down_sample_steps: ${task.obs_down_sample_steps} # float
37
+ # type: low_dim
38
+ # robot0_eef_rot_axis_angle_abs:
39
+ # shape: [2]
40
+ # axis: [0, 1] # yaw and pitch
41
+ # horizon: ${task.low_dim_obs_horizon} # int
42
+ # latency_steps: ${eval:'(${task.camera_obs_latency} - ${task.robot_obs_latency}) * ${task.dataset_frequeny}'} # float
43
+ # down_sample_steps: ${task.obs_down_sample_steps} # float
44
+ # type: low_dim
45
+ robot0_eef_rot_axis_angle:
46
+ raw_shape: [3]
47
+ shape: [6]
48
+ horizon: ${task.low_dim_obs_horizon} # int
49
+ latency_steps: ${eval:'(${task.camera_obs_latency} - ${task.robot_obs_latency}) * ${task.dataset_frequeny}'} # float
50
+ down_sample_steps: ${task.obs_down_sample_steps} # float
51
+ type: low_dim
52
+ rotation_rep: rotation_6d
53
+ ignore_by_policy: ${task.ignore_proprioception}
54
+ robot0_gripper_width:
55
+ shape: [1]
56
+ horizon: ${task.low_dim_obs_horizon} # int
57
+ latency_steps: ${eval:'(${task.camera_obs_latency} - ${task.gripper_obs_latency}) * ${task.dataset_frequeny}'} # float
58
+ down_sample_steps: ${task.obs_down_sample_steps} # float
59
+ type: low_dim
60
+ ignore_by_policy: ${task.ignore_proprioception}
61
+ # Per-frame target bbox in image coordinates: [x0, y0, x1, y1].
62
+ # Bbox shares the camera's clock, so latency_steps stays at 0.
63
+ # Key name must match the zarr array name written by the replay-buffer
64
+ # generation pipeline (currently `bbox`).
65
+ #
66
+ # `bbox_mode` selects how the bbox enters the policy. The value is read
67
+ # by TimmObsEncoder (and UmiDataset's normalizer) and persisted in the
68
+ # checkpoint via Hydra; policy_server serves it back through /shape_meta
69
+ # so eval_real automatically knows which path the model was trained on.
70
+ #
71
+ # low_dim: legacy. The 4 bbox scalars are normalized to [-1, 1]
72
+ # and concatenated to the obs-encoder feature vector
73
+ # alongside the ViT CLS token. Recommended horizon: 2.
74
+ # mask_channel: bbox is rasterized to a 1xHxW binary mask, concatenated
75
+ # with `target_rgb_key` to form a 4-channel image, and
76
+ # fed to the ViT after expanding its patch_embed conv
77
+ # from 3 to 4 input channels (the 4th input slice is
78
+ # zero-initialized at training start). The bbox does
79
+ # NOT also flow through the low-dim concat path in this
80
+ # mode. Recommended horizon: 1 (one mask per image).
81
+ bbox:
82
+ shape: [4] # [x0, y0, x1, y1] in pixel coords of `target_rgb_key`
83
+ horizon: 1 # mask_channel: 1 per image; low_dim: ${task.low_dim_obs_horizon}
84
+ latency_steps: 0 # float perfectly synched with the camera
85
+ down_sample_steps: ${task.obs_down_sample_steps} # same as the camera
86
+ type: low_dim
87
+ ignore_by_policy: False
88
+ bbox_mode: mask_channel # one of: low_dim, mask_channel
89
+ target_rgb_key: camera0_rgb # only used when bbox_mode == mask_channel
90
+
91
+ # pose with respect to episode start
92
+ # robot0_eef_pos_wrt_start:
93
+ # shape: [3]
94
+ # horizon: ${task.low_dim_obs_horizon} # int
95
+ # latency_steps: ${eval:'(${task.camera_obs_latency} - ${task.robot_obs_latency}) * ${task.dataset_frequeny}'} # float
96
+ # down_sample_steps: ${task.obs_down_sample_steps} # float
97
+ # type: low_dim
98
+ # ignore_by_policy: ${task.ignore_proprioception}
99
+ robot0_eef_rot_axis_angle_wrt_start:
100
+ raw_shape: [3]
101
+ shape: [6]
102
+ horizon: ${task.low_dim_obs_horizon} # int
103
+ latency_steps: ${eval:'(${task.camera_obs_latency} - ${task.robot_obs_latency}) * ${task.dataset_frequeny}'} # float
104
+ down_sample_steps: ${task.obs_down_sample_steps} # float
105
+ type: low_dim
106
+ ignore_by_policy: ${task.ignore_proprioception}
107
+ action:
108
+ shape: [10]
109
+ horizon: ${task.action_horizon}
110
+ latency_steps: 0 # float
111
+ down_sample_steps: ${task.obs_down_sample_steps} # int
112
+ rotation_rep: rotation_6d
113
+
114
+ task_name: &task_name umi
115
+ # this will be overridden at call time
116
+ dataset_path: example_demo_session/dataset.zarr.zip
117
+ pose_repr: &pose_repr
118
+ obs_pose_repr: relative # abs or rel
119
+ action_pose_repr: relative # abs or rel or delta
120
+
121
+ env_runner:
122
+ _target_: diffusion_policy.env_runner.real_pusht_image_runner.RealPushTImageRunner
123
+
124
+ dataset:
125
+ _target_: diffusion_policy.dataset.umi_dataset.UmiDataset
126
+ shape_meta: *shape_meta
127
+ dataset_path: ${task.dataset_path}
128
+ cache_dir: null
129
+ pose_repr: *pose_repr
130
+ action_padding: False
131
+ temporally_independent_normalization: False
132
+ repeat_frame_prob: 0.0
133
+ max_duration: null
134
+ seed: 42
135
+ val_ratio: 0.05
@@ -0,0 +1,41 @@
1
+ name: real_image
2
+
3
+ image_shape: [3, 224, 224]
4
+ dataset_path: data/lift_cube_0921
5
+
6
+ shape_meta: &shape_meta
7
+ # acceptable types: rgb, low_dim
8
+ obs:
9
+ camera_0:
10
+ shape: ${task.image_shape}
11
+ type: rgb
12
+ camera_1:
13
+ shape: ${task.image_shape}
14
+ type: rgb
15
+ robot_eef_pose:
16
+ shape: [6]
17
+ type: low_dim
18
+ gripper_position:
19
+ shape: [1]
20
+ type: low_dim
21
+ action:
22
+ shape: [7]
23
+
24
+ env_runner:
25
+ _target_: diffusion_policy.env_runner.real_pusht_image_runner.RealPushTImageRunner
26
+
27
+ dataset:
28
+ _target_: diffusion_policy.dataset.umi_teleop_image_dataset.UmiTeleopImageDataset
29
+ shape_meta: *shape_meta
30
+ dataset_path: ${task.dataset_path}
31
+ horizon: ${horizon}
32
+ pad_before: ${eval:'${n_obs_steps}-1+${n_latency_steps}'}
33
+ pad_after: ${eval:'${horizon}-${n_obs_steps}'}
34
+ n_obs_steps: ${dataset_obs_steps}
35
+ n_latency_steps: ${n_latency_steps}
36
+ use_cache: True
37
+ seed: 42
38
+ val_ratio: 0.00
39
+ max_train_episodes: null
40
+ delta_action: False
41
+
@@ -0,0 +1,185 @@
1
+ defaults:
2
+ - _self_
3
+ - task: umi_bimanual
4
+
5
+ name: train_diffusion_transformer_timm
6
+ _target_: diffusion_policy.workspace.train_diffusion_transformer_timm_workspace.TrainDiffusionTransformerTimmWorkspace
7
+
8
+ task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+
12
+ n_action_steps: 8
13
+ n_emb: 768
14
+
15
+ policy:
16
+ _target_: diffusion_policy.policy.diffusion_transformer_timm_policy.DiffusionTransformerTimmPolicy
17
+
18
+ shape_meta: ${shape_meta}
19
+
20
+ noise_scheduler:
21
+ _target_: diffusers.DDIMScheduler
22
+ num_train_timesteps: 50
23
+ beta_start: 0.0001
24
+ beta_end: 0.02
25
+ # beta_schedule is important
26
+ # this is the best we found
27
+ beta_schedule: squaredcos_cap_v2
28
+ clip_sample: True
29
+ set_alpha_to_one: True
30
+ steps_offset: 0
31
+ prediction_type: epsilon # or sample
32
+
33
+ obs_encoder:
34
+ _target_: diffusion_policy.model.vision.transformer_obs_encoder.TransformerObsEncoder
35
+ shape_meta: ${shape_meta}
36
+ global_pool: ''
37
+ n_emb: ${n_emb}
38
+
39
+ ##### from scratch #####
40
+ # model_name: 'vit_base_patch16_224'
41
+ # model_name: 'resnet34'
42
+ # model_name: 'vit_tiny_patch16_224'
43
+ # model_name: 'efficientnet_b0'
44
+ # model_name: 'efficientnet_b3'
45
+ # pretrained: False
46
+ # frozen: False
47
+
48
+ ##### from scratch #####
49
+ # model_name: 'resnet34.a1_in1k'
50
+ model_name: 'vit_base_patch16_clip_224.openai'
51
+ # model_name: 'convnext_base.clip_laion2b_augreg_ft_in12k'
52
+ pretrained: True
53
+ frozen: False
54
+
55
+ # 'cls' or 'avg' or 'max' or 'soft_attention' or 'spatial_embedding' or 'transformer' or 'attention_pool_2d'
56
+ feature_aggregation: null
57
+
58
+ # it only works for resnet. 32 (7x7) or 16 (14x14)
59
+ downsample_ratio: 32
60
+
61
+ transforms:
62
+ - type: RandomCrop
63
+ ratio: 0.95
64
+ - _target_: torchvision.transforms.RandomRotation
65
+ degrees:
66
+ - -5.0
67
+ - 5.0
68
+ expand: false
69
+ - _target_: torchvision.transforms.ColorJitter
70
+ brightness: 0.3
71
+ contrast: 0.4
72
+ saturation: 0.5
73
+ hue: 0.08
74
+ # - _target_: torchvision.transforms.RandomPerspective
75
+ # distortion_scale: 0.5
76
+ # p: 1.0
77
+ # - _target_: torchvision.transforms.ElasticTransform
78
+ # alpha: 50.0
79
+ # sigma: 5.0
80
+ # - _target_: torchvision.transforms.RandomPerspective
81
+ # distortion_scale: 0.5
82
+ # p: 1.0
83
+ # - _target_: torchvision.transforms.ElasticTransform
84
+ # alpha: 50.0
85
+ # sigma: 5.0
86
+ # - _target_: torchvision.transforms.RandomCrop
87
+ # size: 192
88
+
89
+
90
+ use_group_norm: True
91
+ share_rgb_model: False
92
+
93
+ num_inference_steps: 16
94
+ input_pertub: 0.1 # reference: https://github.com/forever208/DDPM-IP
95
+ n_layer: 7
96
+ n_head: 8
97
+ n_emb: ${n_emb}
98
+ p_drop_attn: 0.1
99
+
100
+
101
+ ema:
102
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
103
+ update_after_step: 0
104
+ inv_gamma: 1.0
105
+ power: 0.75
106
+ min_value: 0.0
107
+ max_value: 0.9999
108
+
109
+ dataloader:
110
+ batch_size: 24
111
+ num_workers: 8
112
+ shuffle: True
113
+ pin_memory: True
114
+ persistent_workers: True
115
+
116
+ val_dataloader:
117
+ batch_size: 24
118
+ num_workers: 8
119
+ shuffle: False
120
+ pin_memory: True
121
+ persistent_workers: True
122
+
123
+ optimizer:
124
+ lr: 3.0e-4
125
+ weight_decay: 1.0e-6
126
+ obs_encoder_lr: 3.0e-5
127
+ obs_encoder_weight_decay: 1.0e-6
128
+ betas: [0.95, 0.999]
129
+
130
+ training:
131
+ device: "cuda:0"
132
+ seed: 42
133
+ debug: False
134
+ resume: False
135
+ # optimization
136
+ lr_scheduler: cosine
137
+ lr_warmup_steps: 2000
138
+ num_epochs: 100
139
+ gradient_accumulate_every: 1
140
+ # EMA destroys performance when used with BatchNorm
141
+ # replace BatchNorm with GroupNorm.
142
+ use_ema: True
143
+ freeze_encoder: False
144
+ # training loop control
145
+ # in epochs
146
+ rollout_every: 10
147
+ checkpoint_every: 10
148
+ val_every: 1
149
+ sample_every: 5
150
+ # steps per epoch
151
+ max_train_steps: null
152
+ max_val_steps: null
153
+ # misc
154
+ tqdm_interval_sec: 1.0
155
+
156
+ logging:
157
+ project: umi
158
+ resume: False
159
+ mode: online
160
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
161
+ tags: ["${name}", "${task_name}", "${exp_name}"]
162
+ id: null
163
+ group: null
164
+
165
+ checkpoint:
166
+ topk:
167
+ monitor_key: train_loss
168
+ mode: min
169
+ k: 20
170
+ format_str: 'epoch_{epoch:04d}-train_loss_{train_loss:.3f}.ckpt'
171
+ save_last_ckpt: True
172
+ save_last_snapshot: False
173
+
174
+ multi_run:
175
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
176
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
177
+
178
+ hydra:
179
+ job:
180
+ override_dirname: ${name}
181
+ run:
182
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
183
+ sweep:
184
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
185
+ subdir: ${hydra.job.num}
@@ -0,0 +1,185 @@
1
+ defaults:
2
+ - _self_
3
+ - task: umi
4
+
5
+ name: train_diffusion_transformer_timm
6
+ _target_: diffusion_policy.workspace.train_diffusion_transformer_timm_workspace.TrainDiffusionTransformerTimmWorkspace
7
+
8
+ task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+
12
+ n_action_steps: 8
13
+ n_emb: 768
14
+
15
+ policy:
16
+ _target_: diffusion_policy.policy.diffusion_transformer_timm_policy.DiffusionTransformerTimmPolicy
17
+
18
+ shape_meta: ${shape_meta}
19
+
20
+ noise_scheduler:
21
+ _target_: diffusers.DDIMScheduler
22
+ num_train_timesteps: 50
23
+ beta_start: 0.0001
24
+ beta_end: 0.02
25
+ # beta_schedule is important
26
+ # this is the best we found
27
+ beta_schedule: squaredcos_cap_v2
28
+ clip_sample: True
29
+ set_alpha_to_one: True
30
+ steps_offset: 0
31
+ prediction_type: epsilon # or sample
32
+
33
+ obs_encoder:
34
+ _target_: diffusion_policy.model.vision.transformer_obs_encoder.TransformerObsEncoder
35
+ shape_meta: ${shape_meta}
36
+ global_pool: ''
37
+ n_emb: ${n_emb}
38
+
39
+ ##### from scratch #####
40
+ # model_name: 'vit_base_patch16_224'
41
+ # model_name: 'resnet34'
42
+ # model_name: 'vit_tiny_patch16_224'
43
+ # model_name: 'efficientnet_b0'
44
+ # model_name: 'efficientnet_b3'
45
+ # pretrained: False
46
+ # frozen: False
47
+
48
+ ##### from scratch #####
49
+ # model_name: 'resnet34.a1_in1k'
50
+ model_name: 'vit_base_patch16_clip_224.openai'
51
+ # model_name: 'convnext_base.clip_laion2b_augreg_ft_in12k'
52
+ pretrained: True
53
+ frozen: False
54
+
55
+ # 'cls' or 'avg' or 'max' or 'soft_attention' or 'spatial_embedding' or 'transformer' or 'attention_pool_2d'
56
+ feature_aggregation: null
57
+
58
+ # it only works for resnet. 32 (7x7) or 16 (14x14)
59
+ downsample_ratio: 32
60
+
61
+ transforms:
62
+ - type: RandomCrop
63
+ ratio: 0.95
64
+ - _target_: torchvision.transforms.RandomRotation
65
+ degrees:
66
+ - -5.0
67
+ - 5.0
68
+ expand: false
69
+ - _target_: torchvision.transforms.ColorJitter
70
+ brightness: 0.3
71
+ contrast: 0.4
72
+ saturation: 0.5
73
+ hue: 0.08
74
+ # - _target_: torchvision.transforms.RandomPerspective
75
+ # distortion_scale: 0.5
76
+ # p: 1.0
77
+ # - _target_: torchvision.transforms.ElasticTransform
78
+ # alpha: 50.0
79
+ # sigma: 5.0
80
+ # - _target_: torchvision.transforms.RandomPerspective
81
+ # distortion_scale: 0.5
82
+ # p: 1.0
83
+ # - _target_: torchvision.transforms.ElasticTransform
84
+ # alpha: 50.0
85
+ # sigma: 5.0
86
+ # - _target_: torchvision.transforms.RandomCrop
87
+ # size: 192
88
+
89
+
90
+ use_group_norm: True
91
+ share_rgb_model: False
92
+
93
+ num_inference_steps: 16
94
+ input_pertub: 0.1 # reference: https://github.com/forever208/DDPM-IP
95
+ n_layer: 7
96
+ n_head: 8
97
+ n_emb: ${n_emb}
98
+ p_drop_attn: 0.1
99
+
100
+
101
+ ema:
102
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
103
+ update_after_step: 0
104
+ inv_gamma: 1.0
105
+ power: 0.75
106
+ min_value: 0.0
107
+ max_value: 0.9999
108
+
109
+ dataloader:
110
+ batch_size: 64
111
+ num_workers: 8
112
+ shuffle: True
113
+ pin_memory: True
114
+ persistent_workers: True
115
+
116
+ val_dataloader:
117
+ batch_size: 64
118
+ num_workers: 8
119
+ shuffle: False
120
+ pin_memory: True
121
+ persistent_workers: True
122
+
123
+ optimizer:
124
+ lr: 3.0e-4
125
+ weight_decay: 1.0e-6
126
+ obs_encoder_lr: 3.0e-5
127
+ obs_encoder_weight_decay: 1.0e-6
128
+ betas: [0.95, 0.999]
129
+
130
+ training:
131
+ device: "cuda:0"
132
+ seed: 42
133
+ debug: False
134
+ resume: False
135
+ # optimization
136
+ lr_scheduler: cosine
137
+ lr_warmup_steps: 2000
138
+ num_epochs: 200
139
+ gradient_accumulate_every: 1
140
+ # EMA destroys performance when used with BatchNorm
141
+ # replace BatchNorm with GroupNorm.
142
+ use_ema: True
143
+ freeze_encoder: False
144
+ # training loop control
145
+ # in epochs
146
+ rollout_every: 10
147
+ checkpoint_every: 10
148
+ val_every: 1
149
+ sample_every: 5
150
+ # steps per epoch
151
+ max_train_steps: null
152
+ max_val_steps: null
153
+ # misc
154
+ tqdm_interval_sec: 1.0
155
+
156
+ logging:
157
+ project: umi
158
+ resume: False
159
+ mode: online
160
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
161
+ tags: ["${name}", "${task_name}", "${exp_name}"]
162
+ id: null
163
+ group: null
164
+
165
+ checkpoint:
166
+ topk:
167
+ monitor_key: train_loss
168
+ mode: min
169
+ k: 20
170
+ format_str: 'epoch_{epoch:04d}-train_loss_{train_loss:.3f}.ckpt'
171
+ save_last_ckpt: True
172
+ save_last_snapshot: False
173
+
174
+ multi_run:
175
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
176
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
177
+
178
+ hydra:
179
+ job:
180
+ override_dirname: ${name}
181
+ run:
182
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
183
+ sweep:
184
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
185
+ subdir: ${hydra.job.num}
@@ -0,0 +1,152 @@
1
+ defaults:
2
+ - _self_
3
+ - task: lift_image_abs
4
+
5
+ name: train_diffusion_unet_image
6
+ _target_: diffusion_policy.workspace.train_diffusion_unet_image_workspace.TrainDiffusionUnetImageWorkspace
7
+
8
+ task_name: ${task.name}
9
+ shape_meta: ${task.shape_meta}
10
+ exp_name: "default"
11
+
12
+ horizon: 16
13
+ n_obs_steps: 2
14
+ n_action_steps: 8
15
+ n_latency_steps: 0
16
+ dataset_obs_steps: ${n_obs_steps}
17
+ past_action_visible: False
18
+ keypoint_visible_rate: 1.0
19
+ obs_as_global_cond: True
20
+
21
+ policy:
22
+ _target_: diffusion_policy.policy.diffusion_unet_image_policy.DiffusionUnetImagePolicy
23
+
24
+ shape_meta: ${shape_meta}
25
+
26
+ noise_scheduler:
27
+ _target_: diffusers.schedulers.scheduling_ddpm.DDPMScheduler
28
+ num_train_timesteps: 100
29
+ beta_start: 0.0001
30
+ beta_end: 0.02
31
+ beta_schedule: squaredcos_cap_v2
32
+ variance_type: fixed_small # Yilun's paper uses fixed_small_log instead, but easy to cause Nan
33
+ clip_sample: True # required when predict_epsilon=False
34
+ prediction_type: epsilon # or sample
35
+
36
+ obs_encoder:
37
+ _target_: diffusion_policy.model.vision.multi_image_obs_encoder.MultiImageObsEncoder
38
+ shape_meta: ${shape_meta}
39
+ rgb_model:
40
+ _target_: diffusion_policy.model.vision.model_getter.get_resnet
41
+ name: resnet18
42
+ weights: null
43
+ resize_shape: null
44
+ crop_shape: [76, 76]
45
+ # crop_shape: [216, 216]
46
+ # constant center crop
47
+ random_crop: True
48
+ use_group_norm: True
49
+ share_rgb_model: False
50
+ imagenet_norm: True
51
+
52
+ horizon: ${horizon}
53
+ n_action_steps: ${eval:'${n_action_steps}+${n_latency_steps}'}
54
+ n_obs_steps: ${n_obs_steps}
55
+ num_inference_steps: 100
56
+ obs_as_global_cond: ${obs_as_global_cond}
57
+ # crop_shape: null
58
+ diffusion_step_embed_dim: 128
59
+ down_dims: [512, 1024, 2048]
60
+ kernel_size: 5
61
+ n_groups: 8
62
+ cond_predict_scale: True
63
+
64
+ # scheduler.step params
65
+ # predict_epsilon: True
66
+
67
+ ema:
68
+ _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
69
+ update_after_step: 0
70
+ inv_gamma: 1.0
71
+ power: 0.75
72
+ min_value: 0.0
73
+ max_value: 0.9999
74
+
75
+ dataloader:
76
+ batch_size: 64
77
+ num_workers: 4
78
+ shuffle: True
79
+ pin_memory: True
80
+ persistent_workers: False
81
+
82
+ val_dataloader:
83
+ batch_size: 64
84
+ num_workers: 4
85
+ # shuffle: False
86
+ shuffle: True
87
+ pin_memory: True
88
+ persistent_workers: False
89
+
90
+ optimizer:
91
+ _target_: torch.optim.AdamW
92
+ lr: 1.0e-4
93
+ betas: [0.95, 0.999]
94
+ eps: 1.0e-8
95
+ weight_decay: 1.0e-6
96
+
97
+ training:
98
+ device: "cuda:0"
99
+ seed: 42
100
+ debug: False
101
+ resume: True
102
+ # optimization
103
+ lr_scheduler: cosine
104
+ lr_warmup_steps: 500
105
+ num_epochs: 8000
106
+ gradient_accumulate_every: 1
107
+ # EMA destroys performance when used with BatchNorm
108
+ # replace BatchNorm with GroupNorm.
109
+ use_ema: True
110
+ freeze_encoder: False
111
+ # training loop control
112
+ # in epochs
113
+ rollout_every: 50
114
+ checkpoint_every: 50
115
+ val_every: 1
116
+ sample_every: 5
117
+ # steps per epoch
118
+ max_train_steps: null
119
+ max_val_steps: null
120
+ # misc
121
+ tqdm_interval_sec: 1.0
122
+
123
+ logging:
124
+ project: diffusion_policy_debug
125
+ resume: True
126
+ mode: online
127
+ name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
128
+ tags: ["${name}", "${task_name}", "${exp_name}"]
129
+ id: null
130
+ group: null
131
+
132
+ checkpoint:
133
+ topk:
134
+ monitor_key: test_mean_score
135
+ mode: max
136
+ k: 5
137
+ format_str: 'epoch_{epoch:04d}-test_mean_score_{test_mean_score:.3f}.ckpt'
138
+ save_last_ckpt: True
139
+ save_last_snapshot: False
140
+
141
+ multi_run:
142
+ run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
143
+ wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
144
+
145
+ hydra:
146
+ job:
147
+ override_dirname: ${name}
148
+ run:
149
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
150
+ sweep:
151
+ dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
152
+ subdir: ${hydra.job.num}