solarian-diffusion 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffusion_policy/__init__.py +0 -0
- diffusion_policy/codecs/__init__.py +0 -0
- diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
- diffusion_policy/common/__init__.py +0 -0
- diffusion_policy/common/checkpoint_util.py +64 -0
- diffusion_policy/common/json_logger.py +117 -0
- diffusion_policy/common/normalize_util.py +251 -0
- diffusion_policy/common/pose_repr_util.py +112 -0
- diffusion_policy/common/pytorch_util.py +82 -0
- diffusion_policy/common/q_critic_util.py +135 -0
- diffusion_policy/common/replay_buffer.py +672 -0
- diffusion_policy/common/sampler.py +204 -0
- diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
- diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
- diffusion_policy/config/legacy/task/can_image.yaml +64 -0
- diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
- diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
- diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
- diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
- diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
- diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
- diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
- diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
- diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
- diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
- diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
- diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
- diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
- diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
- diffusion_policy/config/legacy/task/square_image.yaml +64 -0
- diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
- diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
- diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
- diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
- diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
- diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
- diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
- diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
- diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
- diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
- diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
- diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
- diffusion_policy/config/legacy/task/umi.yaml +45 -0
- diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
- diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
- diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
- diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
- diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
- diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
- diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
- diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
- diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
- diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
- diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
- diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
- diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
- diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
- diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
- diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
- diffusion_policy/config/rl_annotate.yaml +265 -0
- diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
- diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
- diffusion_policy/config/rl_annotate_v2.yaml +141 -0
- diffusion_policy/config/rl_annotate_v3.yaml +165 -0
- diffusion_policy/config/task/lift.yaml +89 -0
- diffusion_policy/config/task/square.yaml +92 -0
- diffusion_policy/config/task/tool_hang.yaml +93 -0
- diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
- diffusion_policy/config/task/umi.yaml +105 -0
- diffusion_policy/config/task/umi_bimanual.yaml +135 -0
- diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
- diffusion_policy/config/task/umi_image.yaml +36 -0
- diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
- diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
- diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
- diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
- diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
- diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
- diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
- diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
- diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
- diffusion_policy/config/train_iql_umi.yaml +186 -0
- diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
- diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
- diffusion_policy/dataset/__init__.py +0 -0
- diffusion_policy/dataset/base_dataset.py +76 -0
- diffusion_policy/dataset/umi_dataset.py +652 -0
- diffusion_policy/env_runner/__init__.py +0 -0
- diffusion_policy/env_runner/base_image_runner.py +9 -0
- diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
- diffusion_policy/model/__init__.py +0 -0
- diffusion_policy/model/common/__init__.py +0 -0
- diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
- diffusion_policy/model/common/lr_scheduler.py +46 -0
- diffusion_policy/model/common/module_attr_mixin.py +14 -0
- diffusion_policy/model/common/normalizer.py +353 -0
- diffusion_policy/model/common/tensor_util.py +960 -0
- diffusion_policy/model/diffusion/__init__.py +0 -0
- diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
- diffusion_policy/model/diffusion/conv1d_components.py +46 -0
- diffusion_policy/model/diffusion/ema_model.py +88 -0
- diffusion_policy/model/diffusion/mask_generator.py +221 -0
- diffusion_policy/model/diffusion/positional_embedding.py +17 -0
- diffusion_policy/model/vision/__init__.py +0 -0
- diffusion_policy/model/vision/crop_randomizer.py +288 -0
- diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
- diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
- diffusion_policy/policy/__init__.py +0 -0
- diffusion_policy/policy/base_image_policy.py +27 -0
- diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
- diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
- diffusion_policy/policy/parl_action_optimizer.py +675 -0
- diffusion_policy/workspace/__init__.py +0 -0
- diffusion_policy/workspace/base_workspace.py +146 -0
- diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
- solarian_diffusion/__init__.py +32 -0
- solarian_diffusion/config.py +31 -0
- solarian_diffusion/policy_server.py +316 -0
- solarian_diffusion/rtc.py +272 -0
- solarian_diffusion/service.py +16 -0
- solarian_diffusion/trainer.py +213 -0
- solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
- solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
- solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
- solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
- solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
- solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
- solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
- umi/__init__.py +0 -0
- umi/common/__init__.py +0 -0
- umi/common/pose_util.py +115 -0
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
# Configuration v2: minimal-prompt annotation against Gemini.
|
|
2
|
+
#
|
|
3
|
+
# Differences vs. `rl_annotate.yaml` (v1):
|
|
4
|
+
# * Calls Gemini 3.5 Flash via the OpenAI compatibility shim
|
|
5
|
+
# (https://generativelanguage.googleapis.com/v1beta/openai/) instead
|
|
6
|
+
# of a local vLLM/MiniCPM/Qwen server.
|
|
7
|
+
# * Single-question prompt — we trust the stronger model and skip the
|
|
8
|
+
# elaborate per-cell-tag scaffolding from v1.
|
|
9
|
+
# * Output schema (`{"outcome": <enum>, "cell_number": <int>}`) is
|
|
10
|
+
# enforced server-side via Gemini's structured-output mode
|
|
11
|
+
# (constrained decoding). The model can only emit tokens that fit
|
|
12
|
+
# the schema, so we don't have to coach it via prompt prose:
|
|
13
|
+
# no "begin with `{`", no "no markdown", no inline JSON examples.
|
|
14
|
+
# * Health check is skipped because Gemini's `/v1/openai/models`
|
|
15
|
+
# listing doesn't match the short model name used here.
|
|
16
|
+
#
|
|
17
|
+
# Usage:
|
|
18
|
+
# export GEMINI_API_KEY=...
|
|
19
|
+
# python scripts/annotate_rewards_vlm.py \
|
|
20
|
+
# --in data/datasets/...zarr.zip \
|
|
21
|
+
# --config diffusion_policy/config/rl_annotate_v2.yaml \
|
|
22
|
+
# --episodes 0 1 2 --visualize
|
|
23
|
+
|
|
24
|
+
vlm:
|
|
25
|
+
base_url: https://generativelanguage.googleapis.com/v1beta/openai/
|
|
26
|
+
# Resolved from the GEMINI_API_KEY env var via OmegaConf interpolation.
|
|
27
|
+
# Override with `--config-overrides vlm.api_key=...` or by editing here.
|
|
28
|
+
api_key: ${oc.env:GEMINI_API_KEY,unset}
|
|
29
|
+
# Stable Gemini Flash slug (per ai.google.dev/gemini-api/docs/models).
|
|
30
|
+
model: gemini-3.5-flash
|
|
31
|
+
temperature: 0.0
|
|
32
|
+
timeout_s: 120.0
|
|
33
|
+
# IMPORTANT: Gemini 3.x defaults to `thinking_level: medium`, which
|
|
34
|
+
# silently consumes thinking tokens out of `max_tokens` BEFORE any
|
|
35
|
+
# visible output. With ~14 images we observed ~250+ thinking tokens
|
|
36
|
+
# per call. We pin `reasoning_effort: low` below to keep this under
|
|
37
|
+
# control, but we still need comfortable headroom for whatever
|
|
38
|
+
# thinking remains plus the schema-bound JSON output (~30 tokens).
|
|
39
|
+
max_tokens: 4096
|
|
40
|
+
# Gemini's mapping of OpenAI's `reasoning_effort` -> `thinking_level`.
|
|
41
|
+
# Allowed values: "minimal" | "low" | "medium" | "high". For this
|
|
42
|
+
# task ("when does the gripper lift the block?") a single forward
|
|
43
|
+
# pass is more than enough — no chain-of-thought needed.
|
|
44
|
+
reasoning_effort: minimal
|
|
45
|
+
# Gemini accepts many images per call natively; no client-side grid
|
|
46
|
+
# composition needed.
|
|
47
|
+
max_images_per_call: null
|
|
48
|
+
# Strict structured output via OpenAI's `json_schema` response_format.
|
|
49
|
+
# Gemini 2.0+ translates this server-side to `responseJsonSchema` and
|
|
50
|
+
# uses constrained decoding to GUARANTEE the reply matches the schema:
|
|
51
|
+
# no preamble, no markdown, no missing/extra keys, no invalid enum
|
|
52
|
+
# values. Because of this, the prompts below can stay terse — the
|
|
53
|
+
# model is physically prevented from emitting anything else.
|
|
54
|
+
response_format:
|
|
55
|
+
type: json_schema
|
|
56
|
+
json_schema:
|
|
57
|
+
name: pick_outcome
|
|
58
|
+
strict: true
|
|
59
|
+
schema:
|
|
60
|
+
type: object
|
|
61
|
+
additionalProperties: false
|
|
62
|
+
required: [outcome, cell_number]
|
|
63
|
+
properties:
|
|
64
|
+
outcome:
|
|
65
|
+
type: string
|
|
66
|
+
enum: [success_correct, wrong_object, miss_empty_grasp]
|
|
67
|
+
cell_number:
|
|
68
|
+
type: integer
|
|
69
|
+
# Hosted endpoint: skip the GET /v1/models verification.
|
|
70
|
+
skip_health_check: true
|
|
71
|
+
|
|
72
|
+
video:
|
|
73
|
+
# Stride for sending frames to the VLM. A 200-frame episode at
|
|
74
|
+
# stride 15 produces ~14 cells, which is plenty for Gemini.
|
|
75
|
+
frame_window_size: 15
|
|
76
|
+
# Hard cap on the number of frames per VLM call. Gemini Flash easily
|
|
77
|
+
# handles 32+ images in one prompt; raise if you want finer
|
|
78
|
+
# localization without chunking.
|
|
79
|
+
# max_keyframes: 32
|
|
80
|
+
max_keyframes: 3200
|
|
81
|
+
# Overlay the per-frame target bbox so the VLM can distinguish the
|
|
82
|
+
# target block from other similar objects.
|
|
83
|
+
bbox_overlay: "all"
|
|
84
|
+
jpeg_quality: 90
|
|
85
|
+
# Image resolution sent to the VLM. Higher = more vision tokens (and
|
|
86
|
+
# higher cost) but better small-detail visibility (gripper-block
|
|
87
|
+
# contact, lift height). 768 → ~1.4k tokens/image; 512 → ~600/image.
|
|
88
|
+
resize_long_side: 768
|
|
89
|
+
|
|
90
|
+
prompts:
|
|
91
|
+
# The schema does the heavy lifting; the prompt only states what the
|
|
92
|
+
# task is. No JSON shape coaching, no formatting rules — Gemini's
|
|
93
|
+
# constrained decoder enforces the response shape directly.
|
|
94
|
+
target_description: "the LEGO block highlighted by the green bounding box"
|
|
95
|
+
system: |
|
|
96
|
+
You are a vision annotator for a robot grasping dataset. Each
|
|
97
|
+
image you receive is one keyframe of a single task execution,
|
|
98
|
+
stamped with a "cell K/N" badge in its top-right corner (K is
|
|
99
|
+
the 0-based cell index, N the total cell count).
|
|
100
|
+
user_template: |
|
|
101
|
+
Identify the earliest cell where the target block (A green
|
|
102
|
+
bounding box marks the target block) is grasped by the gripper and lifted off the surface,
|
|
103
|
+
using the "cell K/N" badges to determine the cell index.
|
|
104
|
+
Answer with key "cell_number" with the number of the cell selected.
|
|
105
|
+
Answer with key "outcome" of the pick attempt: success_correct if the gripper grasped and lifted the bbox-highlighted target block; wrong_object if it grasped a DIFFERENT block; miss_empty_grasp if no block is grasped.
|
|
106
|
+
If no block is grasped, answer cell_number with the value of the last cell.
|
|
107
|
+
|
|
108
|
+
A "ROBOT TELEMETRY" block at the END of this message lists, one
|
|
109
|
+
line per cell, the gripper position (mm relative to the first cell,
|
|
110
|
+
+Z = up) and gripper width (mm) plus the action taken to the next
|
|
111
|
+
cell (dpos, drot_deg, dgripper). A NEGATIVE dgripper paired with
|
|
112
|
+
a POSITIVE dz at cell K is the canonical "closing on the block
|
|
113
|
+
AND lifting" signature — use it as a tie-breaker when the visual
|
|
114
|
+
is occluded.
|
|
115
|
+
reward_map:
|
|
116
|
+
success_correct: 1.0
|
|
117
|
+
wrong_object: -1.0
|
|
118
|
+
miss_empty_grasp: -0.5
|
|
119
|
+
# pile_disturbance: -0.2
|
|
120
|
+
# timeout: 0.0
|
|
121
|
+
# unclear: 0.0
|
|
122
|
+
|
|
123
|
+
io:
|
|
124
|
+
source_zarr: null
|
|
125
|
+
output_zarr: null
|
|
126
|
+
episode_subset: null
|
|
127
|
+
resume: true
|
|
128
|
+
rgb_key: camera0_rgb
|
|
129
|
+
bbox_key: bbox
|
|
130
|
+
# See rl_annotate.yaml for an explanation of the telemetry block.
|
|
131
|
+
# Gemini handles the extra text fine (~50 tokens per cell) and the
|
|
132
|
+
# action delta is the cleanest signal for "did the gripper close
|
|
133
|
+
# on the block and lift it" when the camera view is partial.
|
|
134
|
+
include_proprio_action: true
|
|
135
|
+
eef_pos_key: robot0_eef_pos
|
|
136
|
+
eef_rot_key: robot0_eef_rot_axis_angle
|
|
137
|
+
gripper_key: robot0_gripper_width
|
|
138
|
+
# Strict-by-default: any Gemini call / parse failure aborts the run
|
|
139
|
+
# so we don't silently fill rl/ with bogus labels when an API key is
|
|
140
|
+
# rate-limited / revoked.
|
|
141
|
+
unclear_on_failure: false
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# Configuration v3: minimal-prompt annotation against a LOCAL Qwen2.5-VL.
|
|
2
|
+
#
|
|
3
|
+
# Same prompt-and-schema shape as v2 (terse system prompt, strict
|
|
4
|
+
# `{outcome, cell_number}` JSON schema, telemetry-block tie-breaker)
|
|
5
|
+
# but pointed at a self-hosted vLLM serving Qwen2.5-VL-7B-Instruct
|
|
6
|
+
# instead of Gemini Flash. Compared to v1 (the previous Qwen config)
|
|
7
|
+
# this drops the per-cell binary-tag scaffolding and relies on
|
|
8
|
+
# server-side constrained decoding to keep the output well-formed.
|
|
9
|
+
#
|
|
10
|
+
# Differences vs. v2 (Gemini):
|
|
11
|
+
# * `vlm.base_url` → local vLLM (http://10.0.0.159:8000/v1).
|
|
12
|
+
# * `vlm.model` → Qwen/Qwen2.5-VL-7B-Instruct.
|
|
13
|
+
# * `vlm.api_key` → "local" (vLLM ignores it but the SDK requires
|
|
14
|
+
# a non-empty value).
|
|
15
|
+
# * `reasoning_effort` removed (Qwen2.5-VL has no thinking budget).
|
|
16
|
+
# * `max_tokens` lowered (no thinking-token tax → ~64 tokens of JSON
|
|
17
|
+
# is plenty).
|
|
18
|
+
# * Health check left ENABLED so a down server fails loudly instead
|
|
19
|
+
# of silently flipping every chunk to UNCLEAR.
|
|
20
|
+
#
|
|
21
|
+
# Differences vs. v1 (Qwen, verbose prompt):
|
|
22
|
+
# * No per-cell "before/after" tag procedure. The model only has to
|
|
23
|
+
# answer the two-key schema; the JSON Schema constrained decoder
|
|
24
|
+
# enforces the shape so we don't have to coach formatting.
|
|
25
|
+
# * No reasoning field (the JSON schema forbids extra keys via
|
|
26
|
+
# `additionalProperties: false`).
|
|
27
|
+
#
|
|
28
|
+
# IMPORTANT — vLLM image-cap caveat (same as v1):
|
|
29
|
+
# vLLM defaults to `--limit-mm-per-prompt image=1` even for Qwen2.5-VL.
|
|
30
|
+
# To keep `max_images_per_call: null` below (the recommended setting),
|
|
31
|
+
# you MUST start vLLM with `--limit-mm-per-prompt image=N` where
|
|
32
|
+
# N >= video.max_keyframes. If you see HTTP 400 "At most 1 image(s)
|
|
33
|
+
# may be provided in one prompt", either fix the server flag or set
|
|
34
|
+
# `vlm.max_images_per_call: 1` here.
|
|
35
|
+
#
|
|
36
|
+
# Usage:
|
|
37
|
+
# # On the server side (example):
|
|
38
|
+
# # vllm serve Qwen/Qwen2.5-VL-7B-Instruct \
|
|
39
|
+
# # --port 8000 --limit-mm-per-prompt image=32
|
|
40
|
+
# python scripts/annotate_rewards_vlm.py \
|
|
41
|
+
# --in data/datasets/...zarr.zip \
|
|
42
|
+
# --config diffusion_policy/config/rl_annotate_v3.yaml \
|
|
43
|
+
# --episodes 0 1 2 --visualize
|
|
44
|
+
|
|
45
|
+
vlm:
|
|
46
|
+
base_url: http://10.0.0.159:8000/v1
|
|
47
|
+
api_key: local
|
|
48
|
+
# Qwen2.5-VL-7B-Instruct: solid multi-image understanding, follows
|
|
49
|
+
# structured JSON output reliably under vLLM's guided-decoding
|
|
50
|
+
# backend. Same model as v1; only the prompt scaffolding changes.
|
|
51
|
+
# model: Qwen/Qwen2.5-VL-7B-Instruct
|
|
52
|
+
model: Qwen/Qwen3-VL-4B-Instruct
|
|
53
|
+
temperature: 0.0
|
|
54
|
+
timeout_s: 60.0
|
|
55
|
+
# Two-key strict-schema reply (`{"outcome": "...", "cell_number": N}`)
|
|
56
|
+
# is ~30-40 tokens. 256 leaves comfortable headroom for tokenizer
|
|
57
|
+
# quirks without burning context on dead air.
|
|
58
|
+
max_tokens: 256
|
|
59
|
+
# No client-side grid composition; the server is expected to allow
|
|
60
|
+
# multi-image prompts. See the vLLM caveat at the top of this file
|
|
61
|
+
# if you hit a per-prompt image cap.
|
|
62
|
+
max_images_per_call: null
|
|
63
|
+
# Strict structured output via OpenAI's `json_schema` response_format.
|
|
64
|
+
# vLLM 0.5+ wires this to its guided-decoding backend (xgrammar /
|
|
65
|
+
# outlines), which uses a constrained-decoder to GUARANTEE the
|
|
66
|
+
# reply matches the schema: no preamble, no markdown, no missing /
|
|
67
|
+
# extra keys, no invalid enum values. The prompts below can stay
|
|
68
|
+
# terse because the model is physically prevented from emitting
|
|
69
|
+
# anything else.
|
|
70
|
+
#
|
|
71
|
+
# If your vLLM build is too old to support `response_format:
|
|
72
|
+
# json_schema`, downgrade to `{"type": "json_object"}` and add a
|
|
73
|
+
# one-line "respond with a JSON object" hint to the system prompt.
|
|
74
|
+
response_format:
|
|
75
|
+
type: json_schema
|
|
76
|
+
json_schema:
|
|
77
|
+
name: pick_outcome
|
|
78
|
+
strict: true
|
|
79
|
+
schema:
|
|
80
|
+
type: object
|
|
81
|
+
additionalProperties: false
|
|
82
|
+
required: [outcome, cell_number]
|
|
83
|
+
properties:
|
|
84
|
+
outcome:
|
|
85
|
+
type: string
|
|
86
|
+
enum: [success_correct, wrong_object, miss_empty_grasp]
|
|
87
|
+
cell_number:
|
|
88
|
+
type: integer
|
|
89
|
+
# Local server: keep the GET /v1/models pre-flight enabled so a
|
|
90
|
+
# stopped / mis-loaded vLLM aborts the run immediately instead of
|
|
91
|
+
# silently filling rl/ with UNCLEAR labels.
|
|
92
|
+
skip_health_check: false
|
|
93
|
+
|
|
94
|
+
video:
|
|
95
|
+
# Stride for sending frames to the VLM. A 200-frame episode at
|
|
96
|
+
# stride 15 produces ~14 cells, well within Qwen2.5-VL-7B's
|
|
97
|
+
# multi-image budget.
|
|
98
|
+
frame_window_size: 15
|
|
99
|
+
# Hard cap on frames per VLM call. Qwen2.5-VL-7B can handle 32+
|
|
100
|
+
# images at 768px on a single A100/H100 if the vLLM server is
|
|
101
|
+
# started with a matching `--limit-mm-per-prompt image=N`.
|
|
102
|
+
max_keyframes: 32
|
|
103
|
+
# Overlay the per-frame target bbox so the VLM can distinguish the
|
|
104
|
+
# target block from other similar objects.
|
|
105
|
+
bbox_overlay: "all"
|
|
106
|
+
jpeg_quality: 90
|
|
107
|
+
# Image resolution sent to the VLM. Higher = more vision tokens (and
|
|
108
|
+
# more VRAM / slower decode) but better small-detail visibility
|
|
109
|
+
# (gripper-block contact, lift height). 768 → ~1.4k tokens/image
|
|
110
|
+
# for Qwen2.5-VL; 512 → ~600/image.
|
|
111
|
+
resize_long_side: 768
|
|
112
|
+
|
|
113
|
+
prompts:
|
|
114
|
+
# The schema does the heavy lifting; the prompt only states what the
|
|
115
|
+
# task is. No JSON shape coaching, no formatting rules — vLLM's
|
|
116
|
+
# constrained decoder enforces the response shape directly.
|
|
117
|
+
target_description: "the LEGO block highlighted by the green bounding box"
|
|
118
|
+
system: |
|
|
119
|
+
You are a vision annotator for a robot grasping dataset. Each
|
|
120
|
+
image you receive is one keyframe of a single task execution,
|
|
121
|
+
stamped with a "cell K/N" badge in its top-right corner (K is
|
|
122
|
+
the 0-based cell index, N the total cell count).
|
|
123
|
+
user_template: |
|
|
124
|
+
Identify the earliest cell where the target block (A green
|
|
125
|
+
bounding box marks the target block) is grasped by the gripper and lifted off the surface,
|
|
126
|
+
using the "cell K/N" badges to determine the cell index.
|
|
127
|
+
Answer with key "cell_number" with the number of the cell selected.
|
|
128
|
+
Answer with key "outcome" of the pick attempt: success_correct if the gripper grasped and lifted the bbox-highlighted target block; wrong_object if it grasped a DIFFERENT block; miss_empty_grasp if no block is grasped.
|
|
129
|
+
If no block is grasped, answer cell_number with the value of the last cell.
|
|
130
|
+
|
|
131
|
+
A "ROBOT TELEMETRY" block at the END of this message lists, one
|
|
132
|
+
line per cell, the gripper position (mm relative to the first cell,
|
|
133
|
+
+Z = up) and gripper width (mm) plus the action taken to the next
|
|
134
|
+
cell (dpos, drot_deg, dgripper). A NEGATIVE dgripper paired with
|
|
135
|
+
a POSITIVE dz at cell K is the canonical "closing on the block
|
|
136
|
+
AND lifting" signature — use it as a tie-breaker when the visual
|
|
137
|
+
is occluded.
|
|
138
|
+
|
|
139
|
+
reward_map:
|
|
140
|
+
success_correct: 1.0
|
|
141
|
+
wrong_object: -1.0
|
|
142
|
+
miss_empty_grasp: -0.2
|
|
143
|
+
pile_disturbance: -0.2
|
|
144
|
+
timeout: 0.0
|
|
145
|
+
unclear: 0.0
|
|
146
|
+
|
|
147
|
+
io:
|
|
148
|
+
source_zarr: null
|
|
149
|
+
output_zarr: null
|
|
150
|
+
episode_subset: null
|
|
151
|
+
resume: true
|
|
152
|
+
rgb_key: camera0_rgb
|
|
153
|
+
bbox_key: bbox
|
|
154
|
+
# See rl_annotate.yaml for an explanation of the telemetry block.
|
|
155
|
+
# The action delta is the cleanest signal for "did the gripper
|
|
156
|
+
# close on the block and lift it" when the camera view is partial,
|
|
157
|
+
# so we keep it on by default for the local Qwen config too.
|
|
158
|
+
include_proprio_action: true
|
|
159
|
+
eef_pos_key: robot0_eef_pos
|
|
160
|
+
eef_rot_key: robot0_eef_rot_axis_angle
|
|
161
|
+
gripper_key: robot0_gripper_width
|
|
162
|
+
# Strict-by-default: any VLM call / parse failure aborts the run so
|
|
163
|
+
# we don't silently fill rl/ with bogus labels when the server is
|
|
164
|
+
# OOM / paused / mis-loaded.
|
|
165
|
+
unclear_on_failure: false
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
name: lift_image
|
|
2
|
+
|
|
3
|
+
low_dim_obs_horizon: 2
|
|
4
|
+
img_obs_horizon: 2
|
|
5
|
+
action_horizon: 16
|
|
6
|
+
|
|
7
|
+
shape_meta: &shape_meta
|
|
8
|
+
# acceptable types: rgb, low_dim
|
|
9
|
+
obs:
|
|
10
|
+
# agentview_image:
|
|
11
|
+
# shape: [3, 84, 84]
|
|
12
|
+
# horizon: ${task.img_obs_horizon}
|
|
13
|
+
# type: rgb
|
|
14
|
+
robot0_eye_in_hand_image:
|
|
15
|
+
shape: [3, 84, 84]
|
|
16
|
+
horizon: ${task.img_obs_horizon}
|
|
17
|
+
type: rgb
|
|
18
|
+
robot0_eef_pos_abs:
|
|
19
|
+
shape: [1]
|
|
20
|
+
axis: [2] # 2 means z-axis
|
|
21
|
+
horizon: ${task.low_dim_obs_horizon}
|
|
22
|
+
type: low_dim
|
|
23
|
+
robot0_eef_quat_abs:
|
|
24
|
+
shape: [2]
|
|
25
|
+
axis: [0, 1] # raw and pitch
|
|
26
|
+
horizon: ${task.low_dim_obs_horizon}
|
|
27
|
+
type: low_dim
|
|
28
|
+
robot0_eef_pos:
|
|
29
|
+
shape: [3]
|
|
30
|
+
horizon: ${task.low_dim_obs_horizon}
|
|
31
|
+
type: low_dim
|
|
32
|
+
robot0_eef_quat:
|
|
33
|
+
raw_shape: [4]
|
|
34
|
+
shape: [6]
|
|
35
|
+
horizon: ${task.low_dim_obs_horizon}
|
|
36
|
+
type: low_dim
|
|
37
|
+
rotation_rep: rotation_6d
|
|
38
|
+
robot0_gripper_qpos:
|
|
39
|
+
shape: [2]
|
|
40
|
+
horizon: ${task.low_dim_obs_horizon}
|
|
41
|
+
type: low_dim
|
|
42
|
+
action:
|
|
43
|
+
shape: [10]
|
|
44
|
+
horizon: ${task.action_horizon}
|
|
45
|
+
rotation_rep: rotation_6d
|
|
46
|
+
render:
|
|
47
|
+
agentview_image:
|
|
48
|
+
shape: [3, 84, 84]
|
|
49
|
+
type: rgb
|
|
50
|
+
|
|
51
|
+
task_name: &task_name lift
|
|
52
|
+
dataset_type: &dataset_type ph
|
|
53
|
+
dataset_path: &dataset_path data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
|
|
54
|
+
pose_repr: &pose_repr
|
|
55
|
+
obs_pose_repr: rel # abs or rel
|
|
56
|
+
action_pose_repr: rel
|
|
57
|
+
|
|
58
|
+
env_runner:
|
|
59
|
+
_target_: diffusion_policy.env_runner.robomimic_runner.RobomimicRunner
|
|
60
|
+
dataset_path: *dataset_path
|
|
61
|
+
pose_repr: *pose_repr
|
|
62
|
+
shape_meta: *shape_meta
|
|
63
|
+
n_train: 6
|
|
64
|
+
n_train_vis: 2
|
|
65
|
+
train_start_idx: 0
|
|
66
|
+
n_test: 50
|
|
67
|
+
n_test_vis: 4
|
|
68
|
+
test_start_seed: 100000
|
|
69
|
+
# use python's eval function as resolver, single-quoted string as argument
|
|
70
|
+
max_steps: ${eval:'500 if "${task.dataset_type}" == "mh" else 400'}
|
|
71
|
+
n_action_steps: ${n_action_steps}
|
|
72
|
+
render_obs_key:
|
|
73
|
+
- 'agentview_image'
|
|
74
|
+
- 'robot0_eye_in_hand_image'
|
|
75
|
+
fps: 10
|
|
76
|
+
crf: 22
|
|
77
|
+
tqdm_interval_sec: 1.0
|
|
78
|
+
n_envs: 28
|
|
79
|
+
obs_latency_steps: 0
|
|
80
|
+
# evaluation at this config requires a 16 core 64GB instance.
|
|
81
|
+
|
|
82
|
+
dataset:
|
|
83
|
+
_target_: diffusion_policy.dataset.robomimic_replay_dataset.RobomimicReplayDataset
|
|
84
|
+
shape_meta: *shape_meta
|
|
85
|
+
dataset_path: *dataset_path
|
|
86
|
+
pose_repr: *pose_repr
|
|
87
|
+
temporally_independent_normalization: False
|
|
88
|
+
seed: 42
|
|
89
|
+
val_ratio: 0.02
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
name: square
|
|
2
|
+
|
|
3
|
+
low_dim_obs_horizon: 2
|
|
4
|
+
img_obs_horizon: 2
|
|
5
|
+
action_horizon: 16
|
|
6
|
+
|
|
7
|
+
shape_meta: &shape_meta
|
|
8
|
+
# acceptable types: rgb, low_dim
|
|
9
|
+
obs:
|
|
10
|
+
agentview_image:
|
|
11
|
+
shape: [3, 84, 84]
|
|
12
|
+
horizon: ${task.img_obs_horizon}
|
|
13
|
+
type: rgb
|
|
14
|
+
robot0_eye_in_hand_image:
|
|
15
|
+
shape: [3, 84, 84]
|
|
16
|
+
horizon: ${task.img_obs_horizon}
|
|
17
|
+
type: rgb
|
|
18
|
+
# robot0_eef_pos_abs:
|
|
19
|
+
# shape: [1]
|
|
20
|
+
# axis: [2] # 2 means z-axis
|
|
21
|
+
# horizon: ${task.low_dim_obs_horizon}
|
|
22
|
+
# type: low_dim
|
|
23
|
+
# robot0_eef_quat_abs:
|
|
24
|
+
# shape: [2]
|
|
25
|
+
# axis: [0, 1] # raw and pitch
|
|
26
|
+
# horizon: ${task.low_dim_obs_horizon}
|
|
27
|
+
# type: low_dim
|
|
28
|
+
robot0_eef_pos:
|
|
29
|
+
shape: [3]
|
|
30
|
+
horizon: ${task.low_dim_obs_horizon}
|
|
31
|
+
type: low_dim
|
|
32
|
+
robot0_eef_quat:
|
|
33
|
+
raw_shape: [4]
|
|
34
|
+
shape: [6]
|
|
35
|
+
horizon: ${task.low_dim_obs_horizon}
|
|
36
|
+
type: low_dim
|
|
37
|
+
rotation_rep: rotation_6d
|
|
38
|
+
robot0_gripper_qpos:
|
|
39
|
+
shape: [2]
|
|
40
|
+
horizon: ${task.low_dim_obs_horizon}
|
|
41
|
+
type: low_dim
|
|
42
|
+
action:
|
|
43
|
+
shape: [10]
|
|
44
|
+
horizon: ${task.action_horizon}
|
|
45
|
+
rotation_rep: rotation_6d
|
|
46
|
+
render:
|
|
47
|
+
agentview_image:
|
|
48
|
+
shape: [3, 84, 84]
|
|
49
|
+
type: rgb
|
|
50
|
+
robot0_eye_in_hand_image:
|
|
51
|
+
shape: [3, 84, 84]
|
|
52
|
+
type: rgb
|
|
53
|
+
|
|
54
|
+
task_name: &task_name square
|
|
55
|
+
dataset_type: &dataset_type ph
|
|
56
|
+
dataset_path: &dataset_path data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
|
|
57
|
+
pose_repr: &pose_repr
|
|
58
|
+
obs_pose_repr: abs # abs or rel
|
|
59
|
+
action_pose_repr: abs
|
|
60
|
+
|
|
61
|
+
env_runner:
|
|
62
|
+
_target_: diffusion_policy.env_runner.robomimic_runner.RobomimicRunner
|
|
63
|
+
dataset_path: *dataset_path
|
|
64
|
+
pose_repr: *pose_repr
|
|
65
|
+
shape_meta: *shape_meta
|
|
66
|
+
n_train: 6
|
|
67
|
+
n_train_vis: 2
|
|
68
|
+
train_start_idx: 0
|
|
69
|
+
n_test: 50
|
|
70
|
+
n_test_vis: 4
|
|
71
|
+
test_start_seed: 100000
|
|
72
|
+
# use python's eval function as resolver, single-quoted string as argument
|
|
73
|
+
max_steps: ${eval:'500 if "${task.dataset_type}" == "mh" else 400'}
|
|
74
|
+
n_action_steps: ${n_action_steps}
|
|
75
|
+
render_obs_key:
|
|
76
|
+
- 'agentview_image'
|
|
77
|
+
- 'robot0_eye_in_hand_image'
|
|
78
|
+
fps: 10
|
|
79
|
+
crf: 22
|
|
80
|
+
tqdm_interval_sec: 1.0
|
|
81
|
+
n_envs: 28
|
|
82
|
+
obs_latency_steps: 0
|
|
83
|
+
# evaluation at this config requires a 16 core 64GB instance.
|
|
84
|
+
|
|
85
|
+
dataset:
|
|
86
|
+
_target_: diffusion_policy.dataset.robomimic_replay_dataset.RobomimicReplayDataset
|
|
87
|
+
shape_meta: *shape_meta
|
|
88
|
+
dataset_path: *dataset_path
|
|
89
|
+
pose_repr: *pose_repr
|
|
90
|
+
temporally_independent_normalization: False
|
|
91
|
+
seed: 42
|
|
92
|
+
val_ratio: 0.02
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
name: tool_hang
|
|
2
|
+
|
|
3
|
+
low_dim_obs_horizon: 2
|
|
4
|
+
img_obs_horizon: 2
|
|
5
|
+
action_horizon: 16
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
shape_meta: &shape_meta
|
|
9
|
+
# acceptable types: rgb, low_dim
|
|
10
|
+
obs:
|
|
11
|
+
sideview_image:
|
|
12
|
+
shape: [3, 240, 240]
|
|
13
|
+
horizon: ${task.img_obs_horizon}
|
|
14
|
+
type: rgb
|
|
15
|
+
robot0_eye_in_hand_image:
|
|
16
|
+
shape: [3, 240, 240]
|
|
17
|
+
horizon: ${task.img_obs_horizon}
|
|
18
|
+
type: rgb
|
|
19
|
+
# robot0_eef_pos_abs:
|
|
20
|
+
# shape: [1]
|
|
21
|
+
# axis: [2] # 2 means z-axis
|
|
22
|
+
# horizon: ${task.low_dim_obs_horizon}
|
|
23
|
+
# type: low_dim
|
|
24
|
+
# robot0_eef_quat_abs:
|
|
25
|
+
# shape: [2]
|
|
26
|
+
# axis: [0, 1] # raw and pitch
|
|
27
|
+
# horizon: ${task.low_dim_obs_horizon}
|
|
28
|
+
# type: low_dim
|
|
29
|
+
robot0_eef_pos:
|
|
30
|
+
shape: [3]
|
|
31
|
+
horizon: ${task.low_dim_obs_horizon}
|
|
32
|
+
type: low_dim
|
|
33
|
+
robot0_eef_quat:
|
|
34
|
+
raw_shape: [4]
|
|
35
|
+
shape: [6]
|
|
36
|
+
horizon: ${task.low_dim_obs_horizon}
|
|
37
|
+
type: low_dim
|
|
38
|
+
rotation_rep: rotation_6d
|
|
39
|
+
robot0_gripper_qpos:
|
|
40
|
+
shape: [2]
|
|
41
|
+
horizon: ${task.low_dim_obs_horizon}
|
|
42
|
+
type: low_dim
|
|
43
|
+
action:
|
|
44
|
+
shape: [10]
|
|
45
|
+
horizon: ${task.action_horizon}
|
|
46
|
+
rotation_rep: rotation_6d
|
|
47
|
+
render:
|
|
48
|
+
sideview_image:
|
|
49
|
+
shape: [3, 240, 240]
|
|
50
|
+
type: rgb
|
|
51
|
+
robot0_eye_in_hand_image:
|
|
52
|
+
shape: [3, 240, 240]
|
|
53
|
+
type: rgb
|
|
54
|
+
|
|
55
|
+
task_name: &task_name tool_hang
|
|
56
|
+
dataset_type: &dataset_type ph
|
|
57
|
+
dataset_path: &dataset_path data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
|
|
58
|
+
pose_repr: &pose_repr
|
|
59
|
+
obs_pose_repr: abs # abs or rel
|
|
60
|
+
action_pose_repr: abs
|
|
61
|
+
|
|
62
|
+
env_runner:
|
|
63
|
+
_target_: diffusion_policy.env_runner.robomimic_runner.RobomimicRunner
|
|
64
|
+
dataset_path: *dataset_path
|
|
65
|
+
pose_repr: *pose_repr
|
|
66
|
+
shape_meta: *shape_meta
|
|
67
|
+
n_train: 6
|
|
68
|
+
n_train_vis: 2
|
|
69
|
+
train_start_idx: 0
|
|
70
|
+
n_test: 50
|
|
71
|
+
n_test_vis: 4
|
|
72
|
+
test_start_seed: 100000
|
|
73
|
+
# use python's eval function as resolver, single-quoted string as argument
|
|
74
|
+
max_steps: 700
|
|
75
|
+
n_action_steps: ${n_action_steps}
|
|
76
|
+
render_obs_key:
|
|
77
|
+
- 'sideview_image'
|
|
78
|
+
- 'robot0_eye_in_hand_image'
|
|
79
|
+
fps: 10
|
|
80
|
+
crf: 22
|
|
81
|
+
tqdm_interval_sec: 1.0
|
|
82
|
+
n_envs: 28
|
|
83
|
+
obs_latency_steps: 0
|
|
84
|
+
# evaluation at this config requires a 16 core 64GB instance.
|
|
85
|
+
|
|
86
|
+
dataset:
|
|
87
|
+
_target_: diffusion_policy.dataset.robomimic_replay_dataset.RobomimicReplayDataset
|
|
88
|
+
shape_meta: *shape_meta
|
|
89
|
+
dataset_path: *dataset_path
|
|
90
|
+
pose_repr: *pose_repr
|
|
91
|
+
temporally_independent_normalization: False
|
|
92
|
+
seed: 42
|
|
93
|
+
val_ratio: 0.02
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
name: tool_hang_image_abs
|
|
2
|
+
|
|
3
|
+
shape_meta: &shape_meta
|
|
4
|
+
# acceptable types: rgb, low_dim
|
|
5
|
+
obs:
|
|
6
|
+
sideview_image:
|
|
7
|
+
shape: [3, 240, 240]
|
|
8
|
+
horizon: 2
|
|
9
|
+
type: rgb
|
|
10
|
+
robot0_eye_in_hand_image:
|
|
11
|
+
shape: [3, 240, 240]
|
|
12
|
+
horizon: 2
|
|
13
|
+
type: rgb
|
|
14
|
+
robot0_eef_pos:
|
|
15
|
+
shape: [3]
|
|
16
|
+
type: low_dim
|
|
17
|
+
robot0_eef_quat:
|
|
18
|
+
raw_shape: [4]
|
|
19
|
+
shape: [6]
|
|
20
|
+
horizon: 2
|
|
21
|
+
type: low_dim
|
|
22
|
+
rotation_rep: rotation_6d
|
|
23
|
+
robot0_gripper_qpos:
|
|
24
|
+
shape: [2]
|
|
25
|
+
horizon: 2
|
|
26
|
+
type: low_dim
|
|
27
|
+
action:
|
|
28
|
+
shape: [10]
|
|
29
|
+
rotation_rep: rotation_6d
|
|
30
|
+
|
|
31
|
+
task_name: &task_name tool_hang
|
|
32
|
+
dataset_type: &dataset_type ph
|
|
33
|
+
dataset_path: &dataset_path data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
|
|
34
|
+
pose_repr: &pose_repr
|
|
35
|
+
obs_pose_repr: rel # abs or rel
|
|
36
|
+
action_pose_repr: rel
|
|
37
|
+
|
|
38
|
+
env_runner:
|
|
39
|
+
_target_: diffusion_policy.env_runner.robomimic_runner.RobomimicRunner
|
|
40
|
+
dataset_path: *dataset_path
|
|
41
|
+
pose_repr: *pose_repr
|
|
42
|
+
shape_meta: *shape_meta
|
|
43
|
+
n_train: 6
|
|
44
|
+
n_train_vis: 2
|
|
45
|
+
train_start_idx: 0
|
|
46
|
+
n_test: 50
|
|
47
|
+
n_test_vis: 4
|
|
48
|
+
test_start_seed: 100000
|
|
49
|
+
# use python's eval function as resolver, single-quoted string as argument
|
|
50
|
+
max_steps: ${eval:'500 if "${task.dataset_type}" == "mh" else 400'}
|
|
51
|
+
n_obs_steps: ${n_obs_steps}
|
|
52
|
+
n_action_steps: ${n_action_steps}
|
|
53
|
+
render_obs_key: 'sideview_image'
|
|
54
|
+
fps: 10
|
|
55
|
+
crf: 22
|
|
56
|
+
tqdm_interval_sec: 1.0
|
|
57
|
+
n_envs: 28
|
|
58
|
+
# evaluation at this config requires a 16 core 64GB instance.
|
|
59
|
+
|
|
60
|
+
dataset:
|
|
61
|
+
_target_: diffusion_policy.dataset.robomimic_replay_dataset.RobomimicReplayDataset
|
|
62
|
+
pose_repr: *pose_repr
|
|
63
|
+
shape_meta: *shape_meta
|
|
64
|
+
dataset_path: *dataset_path
|
|
65
|
+
horizon: ${horizon}
|
|
66
|
+
pad_before: ${eval:'${n_obs_steps}-1+${n_latency_steps}'}
|
|
67
|
+
pad_after: ${eval:'${horizon}-${n_obs_steps}'}
|
|
68
|
+
n_obs_steps: ${dataset_obs_steps}
|
|
69
|
+
use_cache: True
|
|
70
|
+
seed: 42
|
|
71
|
+
val_ratio: 0.02
|