solarian-diffusion 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. diffusion_policy/__init__.py +0 -0
  2. diffusion_policy/codecs/__init__.py +0 -0
  3. diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
  4. diffusion_policy/common/__init__.py +0 -0
  5. diffusion_policy/common/checkpoint_util.py +64 -0
  6. diffusion_policy/common/json_logger.py +117 -0
  7. diffusion_policy/common/normalize_util.py +251 -0
  8. diffusion_policy/common/pose_repr_util.py +112 -0
  9. diffusion_policy/common/pytorch_util.py +82 -0
  10. diffusion_policy/common/q_critic_util.py +135 -0
  11. diffusion_policy/common/replay_buffer.py +672 -0
  12. diffusion_policy/common/sampler.py +204 -0
  13. diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
  14. diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
  15. diffusion_policy/config/legacy/task/can_image.yaml +64 -0
  16. diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
  17. diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
  18. diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
  19. diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
  20. diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
  21. diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
  22. diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
  23. diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
  24. diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
  25. diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
  26. diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
  27. diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
  28. diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
  29. diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
  30. diffusion_policy/config/legacy/task/square_image.yaml +64 -0
  31. diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
  32. diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
  33. diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
  34. diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
  35. diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
  36. diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
  37. diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
  38. diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
  39. diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
  40. diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
  41. diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
  42. diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
  43. diffusion_policy/config/legacy/task/umi.yaml +45 -0
  44. diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
  45. diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
  46. diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
  47. diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
  48. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
  49. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
  50. diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
  51. diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
  52. diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
  53. diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
  54. diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
  55. diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
  56. diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
  57. diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
  58. diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
  59. diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
  60. diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
  61. diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
  62. diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
  63. diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
  64. diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
  65. diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
  66. diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
  67. diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
  68. diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
  69. diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
  70. diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
  71. diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
  72. diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
  73. diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
  74. diffusion_policy/config/rl_annotate.yaml +265 -0
  75. diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
  76. diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
  77. diffusion_policy/config/rl_annotate_v2.yaml +141 -0
  78. diffusion_policy/config/rl_annotate_v3.yaml +165 -0
  79. diffusion_policy/config/task/lift.yaml +89 -0
  80. diffusion_policy/config/task/square.yaml +92 -0
  81. diffusion_policy/config/task/tool_hang.yaml +93 -0
  82. diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
  83. diffusion_policy/config/task/umi.yaml +105 -0
  84. diffusion_policy/config/task/umi_bimanual.yaml +135 -0
  85. diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
  86. diffusion_policy/config/task/umi_image.yaml +36 -0
  87. diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
  88. diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
  89. diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
  90. diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
  91. diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
  92. diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
  93. diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
  94. diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
  95. diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
  96. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
  97. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
  98. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
  99. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
  100. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
  101. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
  102. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
  103. diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
  104. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
  105. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
  106. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
  107. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
  108. diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
  109. diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
  110. diffusion_policy/config/train_iql_umi.yaml +186 -0
  111. diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
  112. diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
  113. diffusion_policy/dataset/__init__.py +0 -0
  114. diffusion_policy/dataset/base_dataset.py +76 -0
  115. diffusion_policy/dataset/umi_dataset.py +652 -0
  116. diffusion_policy/env_runner/__init__.py +0 -0
  117. diffusion_policy/env_runner/base_image_runner.py +9 -0
  118. diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
  119. diffusion_policy/model/__init__.py +0 -0
  120. diffusion_policy/model/common/__init__.py +0 -0
  121. diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
  122. diffusion_policy/model/common/lr_scheduler.py +46 -0
  123. diffusion_policy/model/common/module_attr_mixin.py +14 -0
  124. diffusion_policy/model/common/normalizer.py +353 -0
  125. diffusion_policy/model/common/tensor_util.py +960 -0
  126. diffusion_policy/model/diffusion/__init__.py +0 -0
  127. diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
  128. diffusion_policy/model/diffusion/conv1d_components.py +46 -0
  129. diffusion_policy/model/diffusion/ema_model.py +88 -0
  130. diffusion_policy/model/diffusion/mask_generator.py +221 -0
  131. diffusion_policy/model/diffusion/positional_embedding.py +17 -0
  132. diffusion_policy/model/vision/__init__.py +0 -0
  133. diffusion_policy/model/vision/crop_randomizer.py +288 -0
  134. diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
  135. diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
  136. diffusion_policy/policy/__init__.py +0 -0
  137. diffusion_policy/policy/base_image_policy.py +27 -0
  138. diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
  139. diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
  140. diffusion_policy/policy/parl_action_optimizer.py +675 -0
  141. diffusion_policy/workspace/__init__.py +0 -0
  142. diffusion_policy/workspace/base_workspace.py +146 -0
  143. diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
  144. solarian_diffusion/__init__.py +32 -0
  145. solarian_diffusion/config.py +31 -0
  146. solarian_diffusion/policy_server.py +316 -0
  147. solarian_diffusion/rtc.py +272 -0
  148. solarian_diffusion/service.py +16 -0
  149. solarian_diffusion/trainer.py +213 -0
  150. solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
  151. solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
  152. solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
  153. solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
  154. solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
  155. solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
  156. solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
  157. umi/__init__.py +0 -0
  158. umi/common/__init__.py +0 -0
  159. umi/common/pose_util.py +115 -0
@@ -0,0 +1,141 @@
1
+ # Configuration v2: minimal-prompt annotation against Gemini.
2
+ #
3
+ # Differences vs. `rl_annotate.yaml` (v1):
4
+ # * Calls Gemini 3.5 Flash via the OpenAI compatibility shim
5
+ # (https://generativelanguage.googleapis.com/v1beta/openai/) instead
6
+ # of a local vLLM/MiniCPM/Qwen server.
7
+ # * Single-question prompt — we trust the stronger model and skip the
8
+ # elaborate per-cell-tag scaffolding from v1.
9
+ # * Output schema (`{"outcome": <enum>, "cell_number": <int>}`) is
10
+ # enforced server-side via Gemini's structured-output mode
11
+ # (constrained decoding). The model can only emit tokens that fit
12
+ # the schema, so we don't have to coach it via prompt prose:
13
+ # no "begin with `{`", no "no markdown", no inline JSON examples.
14
+ # * Health check is skipped because Gemini's `/v1/openai/models`
15
+ # listing doesn't match the short model name used here.
16
+ #
17
+ # Usage:
18
+ # export GEMINI_API_KEY=...
19
+ # python scripts/annotate_rewards_vlm.py \
20
+ # --in data/datasets/...zarr.zip \
21
+ # --config diffusion_policy/config/rl_annotate_v2.yaml \
22
+ # --episodes 0 1 2 --visualize
23
+
24
+ vlm:
25
+ base_url: https://generativelanguage.googleapis.com/v1beta/openai/
26
+ # Resolved from the GEMINI_API_KEY env var via OmegaConf interpolation.
27
+ # Override with `--config-overrides vlm.api_key=...` or by editing here.
28
+ api_key: ${oc.env:GEMINI_API_KEY,unset}
29
+ # Stable Gemini Flash slug (per ai.google.dev/gemini-api/docs/models).
30
+ model: gemini-3.5-flash
31
+ temperature: 0.0
32
+ timeout_s: 120.0
33
+ # IMPORTANT: Gemini 3.x defaults to `thinking_level: medium`, which
34
+ # silently consumes thinking tokens out of `max_tokens` BEFORE any
35
+ # visible output. With ~14 images we observed ~250+ thinking tokens
36
+ # per call. We pin `reasoning_effort: low` below to keep this under
37
+ # control, but we still need comfortable headroom for whatever
38
+ # thinking remains plus the schema-bound JSON output (~30 tokens).
39
+ max_tokens: 4096
40
+ # Gemini's mapping of OpenAI's `reasoning_effort` -> `thinking_level`.
41
+ # Allowed values: "minimal" | "low" | "medium" | "high". For this
42
+ # task ("when does the gripper lift the block?") a single forward
43
+ # pass is more than enough — no chain-of-thought needed.
44
+ reasoning_effort: minimal
45
+ # Gemini accepts many images per call natively; no client-side grid
46
+ # composition needed.
47
+ max_images_per_call: null
48
+ # Strict structured output via OpenAI's `json_schema` response_format.
49
+ # Gemini 2.0+ translates this server-side to `responseJsonSchema` and
50
+ # uses constrained decoding to GUARANTEE the reply matches the schema:
51
+ # no preamble, no markdown, no missing/extra keys, no invalid enum
52
+ # values. Because of this, the prompts below can stay terse — the
53
+ # model is physically prevented from emitting anything else.
54
+ response_format:
55
+ type: json_schema
56
+ json_schema:
57
+ name: pick_outcome
58
+ strict: true
59
+ schema:
60
+ type: object
61
+ additionalProperties: false
62
+ required: [outcome, cell_number]
63
+ properties:
64
+ outcome:
65
+ type: string
66
+ enum: [success_correct, wrong_object, miss_empty_grasp]
67
+ cell_number:
68
+ type: integer
69
+ # Hosted endpoint: skip the GET /v1/models verification.
70
+ skip_health_check: true
71
+
72
+ video:
73
+ # Stride for sending frames to the VLM. A 200-frame episode at
74
+ # stride 15 produces ~14 cells, which is plenty for Gemini.
75
+ frame_window_size: 15
76
+ # Hard cap on the number of frames per VLM call. Gemini Flash easily
77
+ # handles 32+ images in one prompt; raise if you want finer
78
+ # localization without chunking.
79
+ # max_keyframes: 32
80
+ max_keyframes: 3200
81
+ # Overlay the per-frame target bbox so the VLM can distinguish the
82
+ # target block from other similar objects.
83
+ bbox_overlay: "all"
84
+ jpeg_quality: 90
85
+ # Image resolution sent to the VLM. Higher = more vision tokens (and
86
+ # higher cost) but better small-detail visibility (gripper-block
87
+ # contact, lift height). 768 → ~1.4k tokens/image; 512 → ~600/image.
88
+ resize_long_side: 768
89
+
90
+ prompts:
91
+ # The schema does the heavy lifting; the prompt only states what the
92
+ # task is. No JSON shape coaching, no formatting rules — Gemini's
93
+ # constrained decoder enforces the response shape directly.
94
+ target_description: "the LEGO block highlighted by the green bounding box"
95
+ system: |
96
+ You are a vision annotator for a robot grasping dataset. Each
97
+ image you receive is one keyframe of a single task execution,
98
+ stamped with a "cell K/N" badge in its top-right corner (K is
99
+ the 0-based cell index, N the total cell count).
100
+ user_template: |
101
+ Identify the earliest cell where the target block (A green
102
+ bounding box marks the target block) is grasped by the gripper and lifted off the surface,
103
+ using the "cell K/N" badges to determine the cell index.
104
+ Answer with key "cell_number" with the number of the cell selected.
105
+ Answer with key "outcome" of the pick attempt: success_correct if the gripper grasped and lifted the bbox-highlighted target block; wrong_object if it grasped a DIFFERENT block; miss_empty_grasp if no block is grasped.
106
+ If no block is grasped, answer cell_number with the value of the last cell.
107
+
108
+ A "ROBOT TELEMETRY" block at the END of this message lists, one
109
+ line per cell, the gripper position (mm relative to the first cell,
110
+ +Z = up) and gripper width (mm) plus the action taken to the next
111
+ cell (dpos, drot_deg, dgripper). A NEGATIVE dgripper paired with
112
+ a POSITIVE dz at cell K is the canonical "closing on the block
113
+ AND lifting" signature — use it as a tie-breaker when the visual
114
+ is occluded.
115
+ reward_map:
116
+ success_correct: 1.0
117
+ wrong_object: -1.0
118
+ miss_empty_grasp: -0.5
119
+ # pile_disturbance: -0.2
120
+ # timeout: 0.0
121
+ # unclear: 0.0
122
+
123
+ io:
124
+ source_zarr: null
125
+ output_zarr: null
126
+ episode_subset: null
127
+ resume: true
128
+ rgb_key: camera0_rgb
129
+ bbox_key: bbox
130
+ # See rl_annotate.yaml for an explanation of the telemetry block.
131
+ # Gemini handles the extra text fine (~50 tokens per cell) and the
132
+ # action delta is the cleanest signal for "did the gripper close
133
+ # on the block and lift it" when the camera view is partial.
134
+ include_proprio_action: true
135
+ eef_pos_key: robot0_eef_pos
136
+ eef_rot_key: robot0_eef_rot_axis_angle
137
+ gripper_key: robot0_gripper_width
138
+ # Strict-by-default: any Gemini call / parse failure aborts the run
139
+ # so we don't silently fill rl/ with bogus labels when an API key is
140
+ # rate-limited / revoked.
141
+ unclear_on_failure: false
@@ -0,0 +1,165 @@
1
+ # Configuration v3: minimal-prompt annotation against a LOCAL Qwen2.5-VL.
2
+ #
3
+ # Same prompt-and-schema shape as v2 (terse system prompt, strict
4
+ # `{outcome, cell_number}` JSON schema, telemetry-block tie-breaker)
5
+ # but pointed at a self-hosted vLLM serving Qwen2.5-VL-7B-Instruct
6
+ # instead of Gemini Flash. Compared to v1 (the previous Qwen config)
7
+ # this drops the per-cell binary-tag scaffolding and relies on
8
+ # server-side constrained decoding to keep the output well-formed.
9
+ #
10
+ # Differences vs. v2 (Gemini):
11
+ # * `vlm.base_url` → local vLLM (http://10.0.0.159:8000/v1).
12
+ # * `vlm.model` → Qwen/Qwen2.5-VL-7B-Instruct.
13
+ # * `vlm.api_key` → "local" (vLLM ignores it but the SDK requires
14
+ # a non-empty value).
15
+ # * `reasoning_effort` removed (Qwen2.5-VL has no thinking budget).
16
+ # * `max_tokens` lowered (no thinking-token tax → ~64 tokens of JSON
17
+ # is plenty).
18
+ # * Health check left ENABLED so a down server fails loudly instead
19
+ # of silently flipping every chunk to UNCLEAR.
20
+ #
21
+ # Differences vs. v1 (Qwen, verbose prompt):
22
+ # * No per-cell "before/after" tag procedure. The model only has to
23
+ # answer the two-key schema; the JSON Schema constrained decoder
24
+ # enforces the shape so we don't have to coach formatting.
25
+ # * No reasoning field (the JSON schema forbids extra keys via
26
+ # `additionalProperties: false`).
27
+ #
28
+ # IMPORTANT — vLLM image-cap caveat (same as v1):
29
+ # vLLM defaults to `--limit-mm-per-prompt image=1` even for Qwen2.5-VL.
30
+ # To keep `max_images_per_call: null` below (the recommended setting),
31
+ # you MUST start vLLM with `--limit-mm-per-prompt image=N` where
32
+ # N >= video.max_keyframes. If you see HTTP 400 "At most 1 image(s)
33
+ # may be provided in one prompt", either fix the server flag or set
34
+ # `vlm.max_images_per_call: 1` here.
35
+ #
36
+ # Usage:
37
+ # # On the server side (example):
38
+ # # vllm serve Qwen/Qwen2.5-VL-7B-Instruct \
39
+ # # --port 8000 --limit-mm-per-prompt image=32
40
+ # python scripts/annotate_rewards_vlm.py \
41
+ # --in data/datasets/...zarr.zip \
42
+ # --config diffusion_policy/config/rl_annotate_v3.yaml \
43
+ # --episodes 0 1 2 --visualize
44
+
45
+ vlm:
46
+ base_url: http://10.0.0.159:8000/v1
47
+ api_key: local
48
+ # Qwen2.5-VL-7B-Instruct: solid multi-image understanding, follows
49
+ # structured JSON output reliably under vLLM's guided-decoding
50
+ # backend. Same model as v1; only the prompt scaffolding changes.
51
+ # model: Qwen/Qwen2.5-VL-7B-Instruct
52
+ model: Qwen/Qwen3-VL-4B-Instruct
53
+ temperature: 0.0
54
+ timeout_s: 60.0
55
+ # Two-key strict-schema reply (`{"outcome": "...", "cell_number": N}`)
56
+ # is ~30-40 tokens. 256 leaves comfortable headroom for tokenizer
57
+ # quirks without burning context on dead air.
58
+ max_tokens: 256
59
+ # No client-side grid composition; the server is expected to allow
60
+ # multi-image prompts. See the vLLM caveat at the top of this file
61
+ # if you hit a per-prompt image cap.
62
+ max_images_per_call: null
63
+ # Strict structured output via OpenAI's `json_schema` response_format.
64
+ # vLLM 0.5+ wires this to its guided-decoding backend (xgrammar /
65
+ # outlines), which uses a constrained-decoder to GUARANTEE the
66
+ # reply matches the schema: no preamble, no markdown, no missing /
67
+ # extra keys, no invalid enum values. The prompts below can stay
68
+ # terse because the model is physically prevented from emitting
69
+ # anything else.
70
+ #
71
+ # If your vLLM build is too old to support `response_format:
72
+ # json_schema`, downgrade to `{"type": "json_object"}` and add a
73
+ # one-line "respond with a JSON object" hint to the system prompt.
74
+ response_format:
75
+ type: json_schema
76
+ json_schema:
77
+ name: pick_outcome
78
+ strict: true
79
+ schema:
80
+ type: object
81
+ additionalProperties: false
82
+ required: [outcome, cell_number]
83
+ properties:
84
+ outcome:
85
+ type: string
86
+ enum: [success_correct, wrong_object, miss_empty_grasp]
87
+ cell_number:
88
+ type: integer
89
+ # Local server: keep the GET /v1/models pre-flight enabled so a
90
+ # stopped / mis-loaded vLLM aborts the run immediately instead of
91
+ # silently filling rl/ with UNCLEAR labels.
92
+ skip_health_check: false
93
+
94
+ video:
95
+ # Stride for sending frames to the VLM. A 200-frame episode at
96
+ # stride 15 produces ~14 cells, well within Qwen2.5-VL-7B's
97
+ # multi-image budget.
98
+ frame_window_size: 15
99
+ # Hard cap on frames per VLM call. Qwen2.5-VL-7B can handle 32+
100
+ # images at 768px on a single A100/H100 if the vLLM server is
101
+ # started with a matching `--limit-mm-per-prompt image=N`.
102
+ max_keyframes: 32
103
+ # Overlay the per-frame target bbox so the VLM can distinguish the
104
+ # target block from other similar objects.
105
+ bbox_overlay: "all"
106
+ jpeg_quality: 90
107
+ # Image resolution sent to the VLM. Higher = more vision tokens (and
108
+ # more VRAM / slower decode) but better small-detail visibility
109
+ # (gripper-block contact, lift height). 768 → ~1.4k tokens/image
110
+ # for Qwen2.5-VL; 512 → ~600/image.
111
+ resize_long_side: 768
112
+
113
+ prompts:
114
+ # The schema does the heavy lifting; the prompt only states what the
115
+ # task is. No JSON shape coaching, no formatting rules — vLLM's
116
+ # constrained decoder enforces the response shape directly.
117
+ target_description: "the LEGO block highlighted by the green bounding box"
118
+ system: |
119
+ You are a vision annotator for a robot grasping dataset. Each
120
+ image you receive is one keyframe of a single task execution,
121
+ stamped with a "cell K/N" badge in its top-right corner (K is
122
+ the 0-based cell index, N the total cell count).
123
+ user_template: |
124
+ Identify the earliest cell where the target block (A green
125
+ bounding box marks the target block) is grasped by the gripper and lifted off the surface,
126
+ using the "cell K/N" badges to determine the cell index.
127
+ Answer with key "cell_number" with the number of the cell selected.
128
+ Answer with key "outcome" of the pick attempt: success_correct if the gripper grasped and lifted the bbox-highlighted target block; wrong_object if it grasped a DIFFERENT block; miss_empty_grasp if no block is grasped.
129
+ If no block is grasped, answer cell_number with the value of the last cell.
130
+
131
+ A "ROBOT TELEMETRY" block at the END of this message lists, one
132
+ line per cell, the gripper position (mm relative to the first cell,
133
+ +Z = up) and gripper width (mm) plus the action taken to the next
134
+ cell (dpos, drot_deg, dgripper). A NEGATIVE dgripper paired with
135
+ a POSITIVE dz at cell K is the canonical "closing on the block
136
+ AND lifting" signature — use it as a tie-breaker when the visual
137
+ is occluded.
138
+
139
+ reward_map:
140
+ success_correct: 1.0
141
+ wrong_object: -1.0
142
+ miss_empty_grasp: -0.2
143
+ pile_disturbance: -0.2
144
+ timeout: 0.0
145
+ unclear: 0.0
146
+
147
+ io:
148
+ source_zarr: null
149
+ output_zarr: null
150
+ episode_subset: null
151
+ resume: true
152
+ rgb_key: camera0_rgb
153
+ bbox_key: bbox
154
+ # See rl_annotate.yaml for an explanation of the telemetry block.
155
+ # The action delta is the cleanest signal for "did the gripper
156
+ # close on the block and lift it" when the camera view is partial,
157
+ # so we keep it on by default for the local Qwen config too.
158
+ include_proprio_action: true
159
+ eef_pos_key: robot0_eef_pos
160
+ eef_rot_key: robot0_eef_rot_axis_angle
161
+ gripper_key: robot0_gripper_width
162
+ # Strict-by-default: any VLM call / parse failure aborts the run so
163
+ # we don't silently fill rl/ with bogus labels when the server is
164
+ # OOM / paused / mis-loaded.
165
+ unclear_on_failure: false
@@ -0,0 +1,89 @@
1
+ name: lift_image
2
+
3
+ low_dim_obs_horizon: 2
4
+ img_obs_horizon: 2
5
+ action_horizon: 16
6
+
7
+ shape_meta: &shape_meta
8
+ # acceptable types: rgb, low_dim
9
+ obs:
10
+ # agentview_image:
11
+ # shape: [3, 84, 84]
12
+ # horizon: ${task.img_obs_horizon}
13
+ # type: rgb
14
+ robot0_eye_in_hand_image:
15
+ shape: [3, 84, 84]
16
+ horizon: ${task.img_obs_horizon}
17
+ type: rgb
18
+ robot0_eef_pos_abs:
19
+ shape: [1]
20
+ axis: [2] # 2 means z-axis
21
+ horizon: ${task.low_dim_obs_horizon}
22
+ type: low_dim
23
+ robot0_eef_quat_abs:
24
+ shape: [2]
25
+ axis: [0, 1] # raw and pitch
26
+ horizon: ${task.low_dim_obs_horizon}
27
+ type: low_dim
28
+ robot0_eef_pos:
29
+ shape: [3]
30
+ horizon: ${task.low_dim_obs_horizon}
31
+ type: low_dim
32
+ robot0_eef_quat:
33
+ raw_shape: [4]
34
+ shape: [6]
35
+ horizon: ${task.low_dim_obs_horizon}
36
+ type: low_dim
37
+ rotation_rep: rotation_6d
38
+ robot0_gripper_qpos:
39
+ shape: [2]
40
+ horizon: ${task.low_dim_obs_horizon}
41
+ type: low_dim
42
+ action:
43
+ shape: [10]
44
+ horizon: ${task.action_horizon}
45
+ rotation_rep: rotation_6d
46
+ render:
47
+ agentview_image:
48
+ shape: [3, 84, 84]
49
+ type: rgb
50
+
51
+ task_name: &task_name lift
52
+ dataset_type: &dataset_type ph
53
+ dataset_path: &dataset_path data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
54
+ pose_repr: &pose_repr
55
+ obs_pose_repr: rel # abs or rel
56
+ action_pose_repr: rel
57
+
58
+ env_runner:
59
+ _target_: diffusion_policy.env_runner.robomimic_runner.RobomimicRunner
60
+ dataset_path: *dataset_path
61
+ pose_repr: *pose_repr
62
+ shape_meta: *shape_meta
63
+ n_train: 6
64
+ n_train_vis: 2
65
+ train_start_idx: 0
66
+ n_test: 50
67
+ n_test_vis: 4
68
+ test_start_seed: 100000
69
+ # use python's eval function as resolver, single-quoted string as argument
70
+ max_steps: ${eval:'500 if "${task.dataset_type}" == "mh" else 400'}
71
+ n_action_steps: ${n_action_steps}
72
+ render_obs_key:
73
+ - 'agentview_image'
74
+ - 'robot0_eye_in_hand_image'
75
+ fps: 10
76
+ crf: 22
77
+ tqdm_interval_sec: 1.0
78
+ n_envs: 28
79
+ obs_latency_steps: 0
80
+ # evaluation at this config requires a 16 core 64GB instance.
81
+
82
+ dataset:
83
+ _target_: diffusion_policy.dataset.robomimic_replay_dataset.RobomimicReplayDataset
84
+ shape_meta: *shape_meta
85
+ dataset_path: *dataset_path
86
+ pose_repr: *pose_repr
87
+ temporally_independent_normalization: False
88
+ seed: 42
89
+ val_ratio: 0.02
@@ -0,0 +1,92 @@
1
+ name: square
2
+
3
+ low_dim_obs_horizon: 2
4
+ img_obs_horizon: 2
5
+ action_horizon: 16
6
+
7
+ shape_meta: &shape_meta
8
+ # acceptable types: rgb, low_dim
9
+ obs:
10
+ agentview_image:
11
+ shape: [3, 84, 84]
12
+ horizon: ${task.img_obs_horizon}
13
+ type: rgb
14
+ robot0_eye_in_hand_image:
15
+ shape: [3, 84, 84]
16
+ horizon: ${task.img_obs_horizon}
17
+ type: rgb
18
+ # robot0_eef_pos_abs:
19
+ # shape: [1]
20
+ # axis: [2] # 2 means z-axis
21
+ # horizon: ${task.low_dim_obs_horizon}
22
+ # type: low_dim
23
+ # robot0_eef_quat_abs:
24
+ # shape: [2]
25
+ # axis: [0, 1] # raw and pitch
26
+ # horizon: ${task.low_dim_obs_horizon}
27
+ # type: low_dim
28
+ robot0_eef_pos:
29
+ shape: [3]
30
+ horizon: ${task.low_dim_obs_horizon}
31
+ type: low_dim
32
+ robot0_eef_quat:
33
+ raw_shape: [4]
34
+ shape: [6]
35
+ horizon: ${task.low_dim_obs_horizon}
36
+ type: low_dim
37
+ rotation_rep: rotation_6d
38
+ robot0_gripper_qpos:
39
+ shape: [2]
40
+ horizon: ${task.low_dim_obs_horizon}
41
+ type: low_dim
42
+ action:
43
+ shape: [10]
44
+ horizon: ${task.action_horizon}
45
+ rotation_rep: rotation_6d
46
+ render:
47
+ agentview_image:
48
+ shape: [3, 84, 84]
49
+ type: rgb
50
+ robot0_eye_in_hand_image:
51
+ shape: [3, 84, 84]
52
+ type: rgb
53
+
54
+ task_name: &task_name square
55
+ dataset_type: &dataset_type ph
56
+ dataset_path: &dataset_path data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
57
+ pose_repr: &pose_repr
58
+ obs_pose_repr: abs # abs or rel
59
+ action_pose_repr: abs
60
+
61
+ env_runner:
62
+ _target_: diffusion_policy.env_runner.robomimic_runner.RobomimicRunner
63
+ dataset_path: *dataset_path
64
+ pose_repr: *pose_repr
65
+ shape_meta: *shape_meta
66
+ n_train: 6
67
+ n_train_vis: 2
68
+ train_start_idx: 0
69
+ n_test: 50
70
+ n_test_vis: 4
71
+ test_start_seed: 100000
72
+ # use python's eval function as resolver, single-quoted string as argument
73
+ max_steps: ${eval:'500 if "${task.dataset_type}" == "mh" else 400'}
74
+ n_action_steps: ${n_action_steps}
75
+ render_obs_key:
76
+ - 'agentview_image'
77
+ - 'robot0_eye_in_hand_image'
78
+ fps: 10
79
+ crf: 22
80
+ tqdm_interval_sec: 1.0
81
+ n_envs: 28
82
+ obs_latency_steps: 0
83
+ # evaluation at this config requires a 16 core 64GB instance.
84
+
85
+ dataset:
86
+ _target_: diffusion_policy.dataset.robomimic_replay_dataset.RobomimicReplayDataset
87
+ shape_meta: *shape_meta
88
+ dataset_path: *dataset_path
89
+ pose_repr: *pose_repr
90
+ temporally_independent_normalization: False
91
+ seed: 42
92
+ val_ratio: 0.02
@@ -0,0 +1,93 @@
1
+ name: tool_hang
2
+
3
+ low_dim_obs_horizon: 2
4
+ img_obs_horizon: 2
5
+ action_horizon: 16
6
+
7
+
8
+ shape_meta: &shape_meta
9
+ # acceptable types: rgb, low_dim
10
+ obs:
11
+ sideview_image:
12
+ shape: [3, 240, 240]
13
+ horizon: ${task.img_obs_horizon}
14
+ type: rgb
15
+ robot0_eye_in_hand_image:
16
+ shape: [3, 240, 240]
17
+ horizon: ${task.img_obs_horizon}
18
+ type: rgb
19
+ # robot0_eef_pos_abs:
20
+ # shape: [1]
21
+ # axis: [2] # 2 means z-axis
22
+ # horizon: ${task.low_dim_obs_horizon}
23
+ # type: low_dim
24
+ # robot0_eef_quat_abs:
25
+ # shape: [2]
26
+ # axis: [0, 1] # raw and pitch
27
+ # horizon: ${task.low_dim_obs_horizon}
28
+ # type: low_dim
29
+ robot0_eef_pos:
30
+ shape: [3]
31
+ horizon: ${task.low_dim_obs_horizon}
32
+ type: low_dim
33
+ robot0_eef_quat:
34
+ raw_shape: [4]
35
+ shape: [6]
36
+ horizon: ${task.low_dim_obs_horizon}
37
+ type: low_dim
38
+ rotation_rep: rotation_6d
39
+ robot0_gripper_qpos:
40
+ shape: [2]
41
+ horizon: ${task.low_dim_obs_horizon}
42
+ type: low_dim
43
+ action:
44
+ shape: [10]
45
+ horizon: ${task.action_horizon}
46
+ rotation_rep: rotation_6d
47
+ render:
48
+ sideview_image:
49
+ shape: [3, 240, 240]
50
+ type: rgb
51
+ robot0_eye_in_hand_image:
52
+ shape: [3, 240, 240]
53
+ type: rgb
54
+
55
+ task_name: &task_name tool_hang
56
+ dataset_type: &dataset_type ph
57
+ dataset_path: &dataset_path data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
58
+ pose_repr: &pose_repr
59
+ obs_pose_repr: abs # abs or rel
60
+ action_pose_repr: abs
61
+
62
+ env_runner:
63
+ _target_: diffusion_policy.env_runner.robomimic_runner.RobomimicRunner
64
+ dataset_path: *dataset_path
65
+ pose_repr: *pose_repr
66
+ shape_meta: *shape_meta
67
+ n_train: 6
68
+ n_train_vis: 2
69
+ train_start_idx: 0
70
+ n_test: 50
71
+ n_test_vis: 4
72
+ test_start_seed: 100000
73
+ # use python's eval function as resolver, single-quoted string as argument
74
+ max_steps: 700
75
+ n_action_steps: ${n_action_steps}
76
+ render_obs_key:
77
+ - 'sideview_image'
78
+ - 'robot0_eye_in_hand_image'
79
+ fps: 10
80
+ crf: 22
81
+ tqdm_interval_sec: 1.0
82
+ n_envs: 28
83
+ obs_latency_steps: 0
84
+ # evaluation at this config requires a 16 core 64GB instance.
85
+
86
+ dataset:
87
+ _target_: diffusion_policy.dataset.robomimic_replay_dataset.RobomimicReplayDataset
88
+ shape_meta: *shape_meta
89
+ dataset_path: *dataset_path
90
+ pose_repr: *pose_repr
91
+ temporally_independent_normalization: False
92
+ seed: 42
93
+ val_ratio: 0.02
@@ -0,0 +1,71 @@
1
+ name: tool_hang_image_abs
2
+
3
+ shape_meta: &shape_meta
4
+ # acceptable types: rgb, low_dim
5
+ obs:
6
+ sideview_image:
7
+ shape: [3, 240, 240]
8
+ horizon: 2
9
+ type: rgb
10
+ robot0_eye_in_hand_image:
11
+ shape: [3, 240, 240]
12
+ horizon: 2
13
+ type: rgb
14
+ robot0_eef_pos:
15
+ shape: [3]
16
+ type: low_dim
17
+ robot0_eef_quat:
18
+ raw_shape: [4]
19
+ shape: [6]
20
+ horizon: 2
21
+ type: low_dim
22
+ rotation_rep: rotation_6d
23
+ robot0_gripper_qpos:
24
+ shape: [2]
25
+ horizon: 2
26
+ type: low_dim
27
+ action:
28
+ shape: [10]
29
+ rotation_rep: rotation_6d
30
+
31
+ task_name: &task_name tool_hang
32
+ dataset_type: &dataset_type ph
33
+ dataset_path: &dataset_path data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
34
+ pose_repr: &pose_repr
35
+ obs_pose_repr: rel # abs or rel
36
+ action_pose_repr: rel
37
+
38
+ env_runner:
39
+ _target_: diffusion_policy.env_runner.robomimic_runner.RobomimicRunner
40
+ dataset_path: *dataset_path
41
+ pose_repr: *pose_repr
42
+ shape_meta: *shape_meta
43
+ n_train: 6
44
+ n_train_vis: 2
45
+ train_start_idx: 0
46
+ n_test: 50
47
+ n_test_vis: 4
48
+ test_start_seed: 100000
49
+ # use python's eval function as resolver, single-quoted string as argument
50
+ max_steps: ${eval:'500 if "${task.dataset_type}" == "mh" else 400'}
51
+ n_obs_steps: ${n_obs_steps}
52
+ n_action_steps: ${n_action_steps}
53
+ render_obs_key: 'sideview_image'
54
+ fps: 10
55
+ crf: 22
56
+ tqdm_interval_sec: 1.0
57
+ n_envs: 28
58
+ # evaluation at this config requires a 16 core 64GB instance.
59
+
60
+ dataset:
61
+ _target_: diffusion_policy.dataset.robomimic_replay_dataset.RobomimicReplayDataset
62
+ pose_repr: *pose_repr
63
+ shape_meta: *shape_meta
64
+ dataset_path: *dataset_path
65
+ horizon: ${horizon}
66
+ pad_before: ${eval:'${n_obs_steps}-1+${n_latency_steps}'}
67
+ pad_after: ${eval:'${horizon}-${n_obs_steps}'}
68
+ n_obs_steps: ${dataset_obs_steps}
69
+ use_cache: True
70
+ seed: 42
71
+ val_ratio: 0.02