react-manip 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- assets/materials/realistic/common/ceramic.png +0 -0
- assets/materials/realistic/common/cream-plaster.png +0 -0
- assets/materials/realistic/common/gray-felt.png +0 -0
- assets/materials/realistic/common/gray-plaster.png +0 -0
- assets/materials/realistic/common/gray-woodgrain.png +0 -0
- assets/materials/realistic/common/light-gray-floor-tile.png +0 -0
- assets/materials/realistic/common/light-gray-plaster.png +0 -0
- assets/materials/realistic/common/metal.png +0 -0
- assets/materials/realistic/common/oak-wood.png +0 -0
- assets/materials/realistic/common/steel-brushed.png +0 -0
- assets/materials/realistic/common/steel-scratched.png +0 -0
- assets/materials/realistic/common/varnished-wood-panels.png +0 -0
- assets/materials/realistic/common/walnut-wood.png +0 -0
- assets/materials/realistic/common/white-bricks.png +0 -0
- assets/materials/realistic/common/wood-tiles.png +0 -0
- assets/materials/realistic/tiles074/Tiles074_2K_Color.png +0 -0
- assets/materials/realistic/woodfloor049/WoodFloor049_2K_Color.png +0 -0
- configs/box_insertion/profile_default.json +204 -0
- react_manip/__init__.py +12 -0
- react_manip/asset_cli.py +18 -0
- react_manip/builtin.py +168 -0
- react_manip/cameras.py +301 -0
- react_manip/category_objects.py +248 -0
- react_manip/core/__init__.py +60 -0
- react_manip/core/aggregate.py +59 -0
- react_manip/core/compatibility.py +64 -0
- react_manip/core/contracts.py +42 -0
- react_manip/core/geometry.py +269 -0
- react_manip/core/language.py +116 -0
- react_manip/core/metrics.py +196 -0
- react_manip/core/observation.py +59 -0
- react_manip/core/protocol.py +91 -0
- react_manip/core/registry.py +94 -0
- react_manip/core/runner.py +498 -0
- react_manip/core/scenarios.py +110 -0
- react_manip/core/specs.py +203 -0
- react_manip/core/timeouts.py +44 -0
- react_manip/core/visual_randomization.py +261 -0
- react_manip/evaluation/__init__.py +4 -0
- react_manip/evaluation/metrics.py +17 -0
- react_manip/evaluation/protocols.py +326 -0
- react_manip/evaluation/viewer.py +240 -0
- react_manip/instances/__init__.py +6 -0
- react_manip/instances/candidate_bank.py +109 -0
- react_manip/instances/public.py +95 -0
- react_manip/instances/records.py +345 -0
- react_manip/instances/runtime.py +306 -0
- react_manip/instances/suite.py +231 -0
- react_manip/instances/trajectory.py +59 -0
- react_manip/instances/variation.py +118 -0
- react_manip/legacy.py +318 -0
- react_manip/policies/__init__.py +5 -0
- react_manip/policies/base.py +79 -0
- react_manip/rendering.py +58 -0
- react_manip/robot_registry.py +31 -0
- react_manip/robots/__init__.py +33 -0
- react_manip/robots/adapters.py +135 -0
- react_manip/robots/agilex_adapter.py +35 -0
- react_manip/robots/agilex_description.py +74 -0
- react_manip/robots/agilex_robosuite.py +246 -0
- react_manip/robots/arm_actions.py +17 -0
- react_manip/robots/base.py +119 -0
- react_manip/robots/catalog.py +18 -0
- react_manip/robots/legacy_agilex_robosuite.py +169 -0
- react_manip/robots/mobile_base.py +33 -0
- react_manip/robots/mounts.py +80 -0
- react_manip/robots/resolution.py +103 -0
- react_manip/robots/sawyer_robosuite.py +147 -0
- react_manip/robots/sensors.py +137 -0
- react_manip/utils/__init__.py +1 -0
- react_manip/visual_task.py +50 -0
- react_manip-0.1.0.dist-info/METADATA +446 -0
- react_manip-0.1.0.dist-info/RECORD +99 -0
- react_manip-0.1.0.dist-info/WHEEL +5 -0
- react_manip-0.1.0.dist-info/entry_points.txt +2 -0
- react_manip-0.1.0.dist-info/licenses/LICENSE +131 -0
- react_manip-0.1.0.dist-info/top_level.txt +5 -0
- scripts/add_phone_lean_to_bank.py +242 -0
- scripts/agilex_wheeled_builder.py +133 -0
- scripts/audit_task_solvability.py +150 -0
- scripts/build_agilex_assets.py +317 -0
- scripts/build_legacy_agilex_assets.py +244 -0
- scripts/evaluate_all_tasks.py +309 -0
- scripts/evaluate_policy.py +281 -0
- scripts/piperx_assets.lock.json +181 -0
- scripts/pygame_demo.py +360 -0
- scripts/pygame_task_debug.py +242 -0
- scripts/regenerate_instance_bank.py +192 -0
- scripts/remove_task_from_bank.py +103 -0
- scripts/select_task_instances.py +64 -0
- scripts/verify_curling_reachability.py +206 -0
- scripts/verify_piperx_tasks.py +81 -0
- tasks/box_insertion/__init__.py +3 -0
- tasks/box_insertion/assets/scene.xml +50 -0
- tasks/box_insertion/env.py +642 -0
- tasks/box_insertion/evaluation.json +16 -0
- tasks/box_insertion/instance_contract.json +321 -0
- tasks/box_insertion/scoring.py +23 -0
- tasks/box_insertion/task_spec.json +333 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Task/robot capability checks shared by CLI and runners."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from .specs import TaskSpec
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def resolve_task_robot(spec: TaskSpec, robot_id: str | None = None) -> str | None:
|
|
11
|
+
"""Select the explicit task default, or the sole supported embodiment."""
|
|
12
|
+
if robot_id is None:
|
|
13
|
+
if spec.metadata.get('default_robot') is not None:
|
|
14
|
+
robot_id = spec.metadata['default_robot']
|
|
15
|
+
elif len(spec.supported_robots) == 1:
|
|
16
|
+
robot_id = spec.supported_robots[0]
|
|
17
|
+
elif spec.supported_robots:
|
|
18
|
+
raise ValueError(f"task {spec.task_id!r} has multiple supported robots; specify --robot: "
|
|
19
|
+
+ ', '.join(spec.supported_robots))
|
|
20
|
+
else:
|
|
21
|
+
return None # unrestricted external tasks may own their defaults
|
|
22
|
+
from ..robots.catalog import normalize_robot_id
|
|
23
|
+
robot_id = normalize_robot_id(robot_id)
|
|
24
|
+
validate_task_robot(spec, robot_id)
|
|
25
|
+
return robot_id
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def validate_task_robot(spec: TaskSpec, robot_id: str | None, robot: Any | None = None) -> None:
|
|
29
|
+
"""Raise a descriptive error when a declared combination is unsupported."""
|
|
30
|
+
|
|
31
|
+
if robot_id is None:
|
|
32
|
+
return
|
|
33
|
+
from ..robots.catalog import normalize_robot_id
|
|
34
|
+
normalized = normalize_robot_id(robot_id)
|
|
35
|
+
if not spec.supports_robot(normalized):
|
|
36
|
+
raise ValueError(
|
|
37
|
+
f"task {spec.task_id!r} does not support robot {robot_id!r}; "
|
|
38
|
+
f"supported robots: {', '.join(spec.supported_robots)}"
|
|
39
|
+
)
|
|
40
|
+
if robot is None:
|
|
41
|
+
return
|
|
42
|
+
robot_spec = getattr(robot, "spec", robot)
|
|
43
|
+
capabilities = set(getattr(robot_spec, 'capabilities', ()))
|
|
44
|
+
required = set(spec.metadata.get('required_capabilities', ()))
|
|
45
|
+
if required - capabilities:
|
|
46
|
+
raise ValueError(f'task {spec.task_id!r} requires missing capabilities: {sorted(required - capabilities)}')
|
|
47
|
+
if 'task_level_proxy' in capabilities:
|
|
48
|
+
if spec.metadata.get('embodiment_mode') == 'physical':
|
|
49
|
+
raise ValueError(f'task {spec.task_id!r} requires a physical robot, not a task-level proxy')
|
|
50
|
+
return # Proxy labels cannot attest to physical arm/gripper capabilities.
|
|
51
|
+
arms = getattr(robot_spec, 'arm_names', getattr(robot_spec, 'arms', None))
|
|
52
|
+
arm_count = len(arms) if arms is not None else None
|
|
53
|
+
if arm_count is not None and arm_count < spec.required_arms:
|
|
54
|
+
raise ValueError(
|
|
55
|
+
f"task {spec.task_id!r} requires {spec.required_arms} arms, "
|
|
56
|
+
f"robot {robot_id!r} provides {arm_count}"
|
|
57
|
+
)
|
|
58
|
+
gripper = spec.required_gripper
|
|
59
|
+
actual_gripper = getattr(robot_spec, "gripper_type", None)
|
|
60
|
+
if gripper and actual_gripper and actual_gripper not in {gripper, "robosuite"}:
|
|
61
|
+
raise ValueError(
|
|
62
|
+
f"task {spec.task_id!r} requires gripper {gripper!r}, "
|
|
63
|
+
f"robot {robot_id!r} provides {actual_gripper!r}"
|
|
64
|
+
)
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Machine-checkable observation, action and feedback declarations."""
|
|
2
|
+
from dataclasses import dataclass, field
|
|
3
|
+
from typing import Any, Mapping
|
|
4
|
+
|
|
5
|
+
@dataclass(frozen=True)
|
|
6
|
+
class ObservationSpec:
|
|
7
|
+
modalities: tuple[str, ...] = ("proprio",)
|
|
8
|
+
default_policy_view: tuple[str, ...] = ("proprio",)
|
|
9
|
+
geometry_view: str = "internal"
|
|
10
|
+
fields: Mapping[str, Any] = field(default_factory=dict)
|
|
11
|
+
visibility: Mapping[str, str] = field(default_factory=dict)
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class ActionSpec:
|
|
15
|
+
type: str = "vector"
|
|
16
|
+
shape: tuple[int, ...] = ()
|
|
17
|
+
bounds: tuple[float, float] = (-1.0, 1.0)
|
|
18
|
+
frame: str | None = None
|
|
19
|
+
fields: Mapping[str, Any] = field(default_factory=dict)
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class FeedbackSpec:
|
|
23
|
+
fields: Mapping[str, Any] = field(default_factory=dict)
|
|
24
|
+
events: tuple[str, ...] = ()
|
|
25
|
+
success_signals: tuple[str, ...] = ("success_state",)
|
|
26
|
+
|
|
27
|
+
def parse_contracts(data: Mapping[str, Any]):
|
|
28
|
+
obs=data.get("observation", {}); act=data.get("action", {}); fb=data.get("feedback", {})
|
|
29
|
+
fields=lambda x: x if isinstance(x, Mapping) else {str(v): {} for v in x}
|
|
30
|
+
return (ObservationSpec(tuple(obs.get("modalities", ("proprio",))), tuple(obs.get("default_policy_view", obs.get("modalities", ("proprio",)))), str(obs.get("geometry_view", "internal")), fields(obs.get("fields", {})), dict(obs.get("visibility", {}))),
|
|
31
|
+
ActionSpec(str(act.get("type", "vector")), tuple(act.get("shape", ())), tuple(act.get("bounds", (-1, 1))), act.get("frame"), fields(act.get("fields", {}))),
|
|
32
|
+
FeedbackSpec(fields(fb.get("fields", {})), tuple(fb.get("events", ())), tuple(fb.get("success_signals", ("success_state",)))))
|
|
33
|
+
|
|
34
|
+
# Canonical field vocabulary for tasks whose legacy JSON has not yet been
|
|
35
|
+
# extended. Task-specific specs may override these entries incrementally.
|
|
36
|
+
TASK_CONTRACT_FIELDS = {
|
|
37
|
+
"conveyor_grasp": ("object.pose", "object.velocity", "belt.speed", "grasp_state"),
|
|
38
|
+
"paddle_juggle": ("ball.pose", "ball.velocity", "paddle.pose", "bounce_count"),
|
|
39
|
+
"pitch_pot": ("projectile.pose", "projectile.velocity", "pot.pose", "release_state"),
|
|
40
|
+
"turntable_gamble": ("turntable.angle", "turntable.angular_velocity", "target_sector", "score"),
|
|
41
|
+
"phone_power_on": ("phone.pose", "cup.pose", "button.pose", "press_state"),
|
|
42
|
+
}
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
"""Read-only geometry truth adapters for CL_Coach data collection.
|
|
2
|
+
|
|
3
|
+
The adapters deliberately live outside the policy observation path. They read
|
|
4
|
+
the compiled MuJoCo model/data and task-owned read-only properties; no qpos or
|
|
5
|
+
qvel values are changed. A task can register a specialised adapter without
|
|
6
|
+
changing the benchmark runner.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import Any, ClassVar
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _pose(env: Any, body_id: int) -> dict[str, np.ndarray]:
|
|
18
|
+
d = env.sim.data
|
|
19
|
+
return {
|
|
20
|
+
"position": np.asarray(d.body_xpos[body_id], dtype=float).copy(),
|
|
21
|
+
"quaternion": np.asarray(d.body_xquat[body_id], dtype=float).copy(),
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _eef_pose(env: Any) -> dict[str, np.ndarray] | None:
|
|
26
|
+
try:
|
|
27
|
+
return {"position": np.asarray(env.eef_pos(), float).copy(), "quaternion": np.asarray(env.eef_quat(), float).copy()}
|
|
28
|
+
except Exception:
|
|
29
|
+
return None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class GeometryFeatureAdapter:
|
|
34
|
+
"""Base adapter and task registry.
|
|
35
|
+
|
|
36
|
+
``for_task(env)`` selects an adapter from the task module name. The
|
|
37
|
+
returned mapping is intentionally stable at the top level: entities,
|
|
38
|
+
relations, task_state, validity_mask and schema_version.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
env: Any
|
|
42
|
+
schema_version: ClassVar[str] = "geometry.v1"
|
|
43
|
+
_registry: ClassVar[dict[str, type["GeometryFeatureAdapter"]]] = {}
|
|
44
|
+
|
|
45
|
+
@classmethod
|
|
46
|
+
def register(cls, task_id: str):
|
|
47
|
+
def deco(adapter: type["GeometryFeatureAdapter"]):
|
|
48
|
+
cls._registry[task_id] = adapter
|
|
49
|
+
return adapter
|
|
50
|
+
|
|
51
|
+
return deco
|
|
52
|
+
|
|
53
|
+
@classmethod
|
|
54
|
+
def for_task(cls, env: Any) -> "GeometryFeatureAdapter":
|
|
55
|
+
task_id = getattr(env, "task_id", None) or env.__class__.__module__.split(".")[-2]
|
|
56
|
+
adapter = cls._registry.get(str(task_id))
|
|
57
|
+
# Be convenient for callers that keep the benchmark's legacy wrapper.
|
|
58
|
+
if adapter is None and hasattr(env, "env"):
|
|
59
|
+
env = env.env
|
|
60
|
+
task_id = getattr(env, "task_id", None) or env.__class__.__module__.split(".")[-2]
|
|
61
|
+
adapter = cls._registry.get(str(task_id))
|
|
62
|
+
if adapter is None:
|
|
63
|
+
raise KeyError(f"no geometry adapter registered for task {task_id!r}")
|
|
64
|
+
return adapter(env)
|
|
65
|
+
|
|
66
|
+
def extract(self) -> dict[str, Any]: # pragma: no cover - abstract contract
|
|
67
|
+
raise NotImplementedError
|
|
68
|
+
|
|
69
|
+
def _result(self, entities=None, relations=None, task_state=None, validity_mask=None):
|
|
70
|
+
entities = entities or {}
|
|
71
|
+
relations = relations or {}
|
|
72
|
+
task_state = task_state or {}
|
|
73
|
+
if validity_mask is None:
|
|
74
|
+
validity_mask = {name: True for name in entities}
|
|
75
|
+
return {
|
|
76
|
+
"schema_version": self.schema_version,
|
|
77
|
+
"entities": entities,
|
|
78
|
+
"relations": relations,
|
|
79
|
+
"task_state": task_state,
|
|
80
|
+
"validity_mask": validity_mask,
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@GeometryFeatureAdapter.register("box_insertion")
|
|
85
|
+
class BoxInsertionGeometry(GeometryFeatureAdapter):
|
|
86
|
+
"""Geometric truth for the passive block / active EEF insertion task."""
|
|
87
|
+
|
|
88
|
+
def extract(self):
|
|
89
|
+
e = self.env
|
|
90
|
+
block_pose = e._block_pose()
|
|
91
|
+
eef_pos = np.asarray(e.sim.data.site_xpos[e.eef_site_id], dtype=float).copy()
|
|
92
|
+
entities = {
|
|
93
|
+
"block": {"pose": {"position": block_pose, "quaternion": np.asarray(e.sim.data.xquat[e.block_body_id], dtype=float).copy()}, "velocity": e._block_velocity(), "width_m": np.asarray([e._block_width_m], dtype=float), "patch_friction": e._block_patch_friction.copy(), "patch_width_m": e._block_patch_width_m.copy(), "friction_split_y_m": np.asarray([e._block_friction_split_y_m], dtype=float), "friction_split_from_left_m": np.asarray([e._block_friction_split_from_left_m], dtype=float)},
|
|
94
|
+
"box": {"entry_x": np.asarray([e._slot_bounds[0]], dtype=float), "bounds_xy": np.asarray(e._slot_bounds, dtype=float), "opening_width_m": np.asarray([e._slot_opening_width_m], dtype=float), "clearance_m": np.asarray([e._slot_clearance_m], dtype=float), "target_depth": np.asarray([e._TARGET_DEPTH], dtype=float)},
|
|
95
|
+
"table": {"top_z": np.asarray([0.78], dtype=float)},
|
|
96
|
+
"eef": {"pose": {"position": eef_pos, "matrix": np.asarray(e.sim.data.site_xmat[e.eef_site_id], dtype=float).copy()}},
|
|
97
|
+
}
|
|
98
|
+
relations = {
|
|
99
|
+
"eef_to_push_face": np.asarray(block_pose - eef_pos + np.array([-e._block_half_length, 0, 0]), dtype=float),
|
|
100
|
+
"remaining_insertion_distance": np.asarray([max(0.0, e._TARGET_DEPTH - e._block_depth())], dtype=float),
|
|
101
|
+
"contact_force_N": np.asarray([e._contact_force()], dtype=float),
|
|
102
|
+
}
|
|
103
|
+
state = {"insertion_depth_m": float(e._block_depth()), "block_contained": bool(e._block_contained()), "bottom_reached": bool(e._bottom_reached()), "success_blockers": e._success_blockers(), "success": bool(e._events), "failure": e._failure_reason}
|
|
104
|
+
return self._result(entities, relations, state, {"block": True, "box": True, "table": True, "eef": True})
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@GeometryFeatureAdapter.register("laptop_into_drawer")
|
|
108
|
+
class LaptopIntoDrawerGeometry(GeometryFeatureAdapter):
|
|
109
|
+
def extract(self):
|
|
110
|
+
e = self.env
|
|
111
|
+
entities = {"laptop": {"pose": _pose(e, e.laptop_body_id)}, "handle": {"pose": {"position": np.asarray(e.sim.data.site_xpos[e.handle_site_id], float).copy()}}}
|
|
112
|
+
entities["drawer"] = {"slide_position": np.asarray([e.sim.data.qpos[e.drawer_qpos_addr]], float)}
|
|
113
|
+
if hasattr(e, 'geometry'):
|
|
114
|
+
entities['laptop']['size'] = np.asarray(e.laptop_size,float).copy()
|
|
115
|
+
entities['drawer'].update(inner_size=np.array([e.geometry.depth,e.geometry.width,e.geometry.height]),
|
|
116
|
+
max_open_m=e.geometry.opening, floor_z=e.geometry.floor)
|
|
117
|
+
ee = _eef_pose(e)
|
|
118
|
+
if ee:
|
|
119
|
+
entities["eef"] = {"pose": ee}
|
|
120
|
+
entities["eef"]["velocity"] = np.asarray(getattr(e, "eef_vel", lambda: np.zeros(3))(), float).copy()
|
|
121
|
+
bounds = np.asarray(e._drawer_inner_bounds(), float)
|
|
122
|
+
drawer_center = np.asarray([(bounds[0] + bounds[1]) / 2.0, (bounds[2] + bounds[3]) / 2.0, e.drawer_floor_z + e.geometry.height/2], float)
|
|
123
|
+
relations = {"drawer.inner_bounds": bounds, "laptop_to_drawer": drawer_center - entities["laptop"]["pose"]["position"]}
|
|
124
|
+
if ee:
|
|
125
|
+
relations["eef_to_handle"] = entities["handle"]["pose"]["position"] - ee["position"]
|
|
126
|
+
return self._result(entities, relations, {"success": bool(e._check_success())}, {"laptop": True, "drawer": True})
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@GeometryFeatureAdapter.register("conveyor_grasp")
|
|
130
|
+
class ConveyorGraspGeometry(GeometryFeatureAdapter):
|
|
131
|
+
def extract(self):
|
|
132
|
+
e, d = self.env, self.env.sim.data
|
|
133
|
+
objects = []
|
|
134
|
+
for i, bid in enumerate(e.obj_body_ids):
|
|
135
|
+
objects.append({"id": i, "pose": _pose(e, int(bid)), "velocity": np.asarray(e.obj_vel(i), float).copy(),
|
|
136
|
+
"category_id": e.records[i]["category"], "variant_id": e.records[i]["variant"],
|
|
137
|
+
"extent": e.records[i]["extent"].copy(), "phase": str(e.phases[i])})
|
|
138
|
+
entities = {"objects": objects, "eef": {"pose": {"position": np.asarray(e.eef_pos(), float).copy()}}}
|
|
139
|
+
entities["belt"] = {"phase": np.asarray([d.qpos[e.belt_qpos_addr]], float), "speed": np.asarray([getattr(e, "_v_belt", 0.0)], float)}
|
|
140
|
+
from tasks.conveyor_grasp.env import BIN_HALF
|
|
141
|
+
entities["baskets"] = [{"id": i, "position": c.copy(), "half_size": BIN_HALF.copy(),
|
|
142
|
+
"bounds": np.stack([c + [-BIN_HALF[0], -BIN_HALF[1], .01], c + [BIN_HALF[0], BIN_HALF[1], 2 * BIN_HALF[2]]]),
|
|
143
|
+
"category_id": int(e.bin_categories[i]), "fixed_category_id": int(e.fixed_categories[i])}
|
|
144
|
+
for i, c in enumerate(e.bin_centers)]
|
|
145
|
+
entities["examples"] = [{"pose": _pose(e, int(b)), "category_id": r["category"], "basket_id": r["basket"]}
|
|
146
|
+
for b,r in zip(e.example_body_ids,e.examples)]
|
|
147
|
+
relations = {"object_basket": e._bin_ids.copy(), "correct_mask": e._counted.copy(), "mixed_baskets": e._mixed.copy()}
|
|
148
|
+
return self._result(entities, relations, {"score": e.total_score, "time": float(d.time), "examples_retained": e._examples_retained})
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@GeometryFeatureAdapter.register("phone_lean")
|
|
156
|
+
class PhoneLeanGeometry(GeometryFeatureAdapter):
|
|
157
|
+
"""Read-only geometry snapshot for the phone-lean scene."""
|
|
158
|
+
|
|
159
|
+
def extract(self):
|
|
160
|
+
e, d = self.env, self.env.sim.data
|
|
161
|
+
def body_pose(body_id):
|
|
162
|
+
pose = _pose(e, body_id)
|
|
163
|
+
pose["matrix"] = np.asarray(d.body_xmat[body_id], float).reshape(3, 3).copy()
|
|
164
|
+
return pose
|
|
165
|
+
entities = {name: {"pose": body_pose(getattr(e, name+'_body_id'))}
|
|
166
|
+
for name in ('phone', *e.support_targets)}
|
|
167
|
+
ee_by_arm = {}
|
|
168
|
+
for arm in ('right', 'left'):
|
|
169
|
+
if arm not in e.robots[0].eef_site_id:
|
|
170
|
+
continue
|
|
171
|
+
site = e.robots[0].eef_site_id[arm]
|
|
172
|
+
ee_by_arm[arm] = {'position': d.site_xpos[site].copy(),
|
|
173
|
+
'matrix': d.site_xmat[site].reshape(3, 3).copy()}
|
|
174
|
+
entities[f"eef_{arm}"] = {"pose": ee_by_arm[arm]}
|
|
175
|
+
if 'right' in ee_by_arm:
|
|
176
|
+
entities["eef"] = {"pose": ee_by_arm['right']}
|
|
177
|
+
phone = entities["phone"]["pose"]["position"]
|
|
178
|
+
supports = {name: entities[name]["pose"]["position"] for name in e.support_targets}
|
|
179
|
+
relations = {
|
|
180
|
+
"eef_to_phone": (phone - ee_by_arm['right']["position"]
|
|
181
|
+
if 'right' in ee_by_arm else np.zeros(3)),
|
|
182
|
+
"eef_to_phone_by_arm": {
|
|
183
|
+
arm: phone - pose["position"] for arm, pose in ee_by_arm.items()
|
|
184
|
+
},
|
|
185
|
+
"phone_to_supports": {name: position - phone for name, position in supports.items()},
|
|
186
|
+
}
|
|
187
|
+
distance, height = e._phone_metrics()
|
|
188
|
+
state = {
|
|
189
|
+
"phone_lean_xy_error": float(distance),
|
|
190
|
+
"phone_height": float(height),
|
|
191
|
+
"success": bool(distance < 0.06 and height > 0.72),
|
|
192
|
+
}
|
|
193
|
+
validity = {name: True for name in entities}
|
|
194
|
+
return self._result(entities, relations, state, validity)
|
|
195
|
+
@GeometryFeatureAdapter.register("pitch_pot")
|
|
196
|
+
class PitchPotGeometry(GeometryFeatureAdapter):
|
|
197
|
+
def extract(self):
|
|
198
|
+
e = self.env
|
|
199
|
+
from tasks.pitch_pot.env import TRAY_POS
|
|
200
|
+
velocity = np.asarray(e.sim.data.qvel[e.obj_qvel_addr:e.obj_qvel_addr + 3], float).copy()
|
|
201
|
+
entities = {"projectile": {"pose": _pose(e, e.obj_body_id), "velocity": velocity}, "pot": {"pose": _pose(e, e.pot_body_id)}, "tray": {"position": np.asarray(TRAY_POS, float).copy()}}
|
|
202
|
+
in_state = bool(e._obj_in_pot())
|
|
203
|
+
return self._result(entities, {"projectile_to_pot": entities["pot"]["pose"]["position"] - entities["projectile"]["pose"]["position"]}, {"in_state": in_state, "release_state": "in_pot" if in_state else "free", "attempts": int(e.attempts), "hits": int(e.hits)})
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
@GeometryFeatureAdapter.register("button_indicator_matching")
|
|
209
|
+
class ButtonIndicatorMatchingGeometry(GeometryFeatureAdapter):
|
|
210
|
+
"""Read-only geometry truth for the button / indicator event task.
|
|
211
|
+
|
|
212
|
+
The button-to-indicator permutation is included for oracle and benchmark
|
|
213
|
+
logs. Consumers exposing policy observations should filter this field
|
|
214
|
+
according to the task specification's ``geometry_view`` contract.
|
|
215
|
+
"""
|
|
216
|
+
|
|
217
|
+
def extract(self):
|
|
218
|
+
e = self.env
|
|
219
|
+
d = e.sim.data
|
|
220
|
+
pressed = np.asarray(e._pressed(), dtype=bool)
|
|
221
|
+
indicator_on = np.zeros(int(e.n_buttons), dtype=bool)
|
|
222
|
+
indicator_on[np.asarray(e.assignment, dtype=int)[pressed]] = True
|
|
223
|
+
button_positions = np.asarray(d.qpos[e._button_qadr], dtype=float).copy()
|
|
224
|
+
button_entities = []
|
|
225
|
+
for i, position in enumerate(button_positions):
|
|
226
|
+
body_id = e.sim.model.body_name2id(f"button_{i}")
|
|
227
|
+
button_entities.append({
|
|
228
|
+
"id": i,
|
|
229
|
+
"pose": _pose(e, body_id),
|
|
230
|
+
"deflection_m": float(position),
|
|
231
|
+
"pressed": bool(pressed[i]),
|
|
232
|
+
# Buttons are intentionally unlabeled; labels belong to the
|
|
233
|
+
# indicator side of the random wiring.
|
|
234
|
+
"label": None,
|
|
235
|
+
})
|
|
236
|
+
indicator_entities = []
|
|
237
|
+
for i in range(int(e.n_buttons)):
|
|
238
|
+
gid = e._indicator_gids[i]
|
|
239
|
+
indicator_position = np.asarray(d.geom_xpos[gid], dtype=float).copy()
|
|
240
|
+
indicator_entities.append({
|
|
241
|
+
"id": i,
|
|
242
|
+
"label": str(e.labels[i]),
|
|
243
|
+
"color": str(e.labels[i]) if getattr(e, "label_mode", "") == "colors" else "green",
|
|
244
|
+
"on": bool(indicator_on[i]),
|
|
245
|
+
"pose": {"position": indicator_position,
|
|
246
|
+
"matrix": np.asarray(d.geom_xmat[gid], dtype=float).copy()},
|
|
247
|
+
"position": indicator_position,
|
|
248
|
+
})
|
|
249
|
+
entities = {
|
|
250
|
+
"buttons": button_entities,
|
|
251
|
+
"indicators": indicator_entities,
|
|
252
|
+
}
|
|
253
|
+
ee = _eef_pose(e)
|
|
254
|
+
if ee:
|
|
255
|
+
entities["eef"] = {"pose": ee}
|
|
256
|
+
assignment = np.asarray(e.assignment, dtype=np.int32).copy()
|
|
257
|
+
relations = {
|
|
258
|
+
"button_to_indicator": assignment,
|
|
259
|
+
"target_indicator": np.asarray([int(e._target_indicator)], dtype=np.int32),
|
|
260
|
+
"target_label": str(e.labels[e._target_indicator]),
|
|
261
|
+
}
|
|
262
|
+
state = {
|
|
263
|
+
"score": int(e._score),
|
|
264
|
+
"events": list(getattr(e, "_events", ())),
|
|
265
|
+
"elapsed_sec": float(d.time),
|
|
266
|
+
"label_mode": str(e.label_mode),
|
|
267
|
+
}
|
|
268
|
+
validity = {"buttons": True, "indicators": True, "eef": ee is not None}
|
|
269
|
+
return self._result(entities, relations, state, validity)
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Episode-level natural-language task prompt generation."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from dataclasses import asdict, dataclass
|
|
5
|
+
from typing import Any, Mapping
|
|
6
|
+
|
|
7
|
+
from .specs import TaskSpec
|
|
8
|
+
|
|
9
|
+
def normalize_language_prompt(value: Mapping[str, Any]) -> dict[str, Any]:
|
|
10
|
+
"""Add explicit task / current-round scopes without breaking legacy consumers.
|
|
11
|
+
|
|
12
|
+
A round is a subgoal within an episode, not another environment reset.
|
|
13
|
+
Tasks without subgoals leave round_instruction empty and round_index None.
|
|
14
|
+
Legacy instruction/text retain their existing combined representation.
|
|
15
|
+
"""
|
|
16
|
+
result = dict(value)
|
|
17
|
+
result.setdefault("task_instruction", str(result.get("instruction", "")))
|
|
18
|
+
result.setdefault("round_instruction", "")
|
|
19
|
+
result.setdefault("round_index", None)
|
|
20
|
+
combined = " ".join(str(result[key]) for key in ("task_instruction", "round_instruction") if result[key])
|
|
21
|
+
result.setdefault("instruction", combined)
|
|
22
|
+
result.setdefault("text", combined)
|
|
23
|
+
return result
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class EpisodeLanguagePrompt:
|
|
28
|
+
task_id: str
|
|
29
|
+
instruction: str
|
|
30
|
+
robot_id: str | None
|
|
31
|
+
episode_index: int
|
|
32
|
+
scenario_id: str | None
|
|
33
|
+
success_condition: str
|
|
34
|
+
constraints: tuple[str, ...] = ()
|
|
35
|
+
public_context: Mapping[str, Any] | None = None
|
|
36
|
+
task_instruction: str = ""
|
|
37
|
+
round_instruction: str = ""
|
|
38
|
+
round_index: int | None = None
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def text(self) -> str:
|
|
42
|
+
robot = self.robot_id or "unspecified embodiment"
|
|
43
|
+
context = ""
|
|
44
|
+
if self.public_context:
|
|
45
|
+
context = " Public scene context: " + ", ".join(
|
|
46
|
+
f"{k}={v}" for k, v in self.public_context.items()
|
|
47
|
+
) + "."
|
|
48
|
+
limits = "" if not self.constraints else " Constraints: " + "; ".join(self.constraints) + "."
|
|
49
|
+
current = f" Current round: {self.round_instruction}" if self.round_instruction else ""
|
|
50
|
+
return f"Task: {self.task_instruction or self.instruction}{current} Embodiment: {robot}. Success: {self.success_condition}.{context}{limits}"
|
|
51
|
+
|
|
52
|
+
def to_dict(self) -> dict[str, Any]:
|
|
53
|
+
value = asdict(self)
|
|
54
|
+
value["constraints"] = list(self.constraints)
|
|
55
|
+
value["public_context"] = dict(self.public_context or {})
|
|
56
|
+
value["text"] = self.text
|
|
57
|
+
value["task_instruction"] = self.task_instruction or self.instruction
|
|
58
|
+
return normalize_language_prompt(value)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _robot_id_from_env(env: Any) -> str | None:
|
|
62
|
+
if getattr(env, "robot_id", None):
|
|
63
|
+
return str(env.robot_id)
|
|
64
|
+
robots = getattr(env, "robots", ()) or ()
|
|
65
|
+
if not robots:
|
|
66
|
+
return None
|
|
67
|
+
name = str(getattr(robots[0], "name", "")).lower()
|
|
68
|
+
from ..robots.resolution import robot_id_from_model
|
|
69
|
+
try:return robot_id_from_model(name)
|
|
70
|
+
except ValueError:pass
|
|
71
|
+
for key, marker in (("aloha_mobile", "agilexmobilealoha"), ("aloha", "agilexaloha"), ("legacy_piperx", "agilexpiper"), ("mobile_dual_piperx", "mobiledualpiperx"), ("dual_piperx", "dualpiperx"),
|
|
72
|
+
("piperx", "piperx"), ("sawyer", "sawyer"), ("panda", "panda")):
|
|
73
|
+
if marker in name:
|
|
74
|
+
return key
|
|
75
|
+
return None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def task_language_prompt(env: Any, *, task_id: str, task_instruction: str,
|
|
79
|
+
success_condition: str, constraints: tuple[str, ...] = ()) -> dict[str, Any]:
|
|
80
|
+
"""Format rules owned by a task's runtime environment, never by JSON."""
|
|
81
|
+
return EpisodeLanguagePrompt(
|
|
82
|
+
task_id=task_id, instruction=task_instruction, task_instruction=task_instruction,
|
|
83
|
+
robot_id=_robot_id_from_env(env),
|
|
84
|
+
episode_index=max(0, int(getattr(env, "_episode_index", 0))),
|
|
85
|
+
scenario_id=getattr(getattr(env, "scenario", None), "scenario_id", None),
|
|
86
|
+
success_condition=success_condition, constraints=constraints,
|
|
87
|
+
).to_dict()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def runtime_language_prompt(env: Any) -> dict[str, Any]:
|
|
91
|
+
"""Require the task-owned runtime interface; missing rules are an error."""
|
|
92
|
+
method = getattr(env, "get_episode_language_prompt", None)
|
|
93
|
+
if not callable(method):
|
|
94
|
+
raise RuntimeError(f"{type(env).__name__} must implement get_episode_language_prompt()")
|
|
95
|
+
value = method()
|
|
96
|
+
if not isinstance(value, Mapping):
|
|
97
|
+
raise TypeError("get_episode_language_prompt() must return a mapping")
|
|
98
|
+
prompt = normalize_language_prompt(value)
|
|
99
|
+
if not str(prompt["task_instruction"]).strip() or not str(prompt["text"]).strip():
|
|
100
|
+
raise ValueError("Runtime language prompt must contain non-empty task_instruction and text")
|
|
101
|
+
return prompt
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def build_episode_prompt(spec: TaskSpec, env: Any = None, *, robot_id: str | None = None,
|
|
105
|
+
scenario: Any = None, episode_index: int = 0) -> EpisodeLanguagePrompt:
|
|
106
|
+
"""Compatibility formatter using runtime rules only; JSON is never a fallback."""
|
|
107
|
+
prompt = runtime_language_prompt(env)
|
|
108
|
+
return EpisodeLanguagePrompt(
|
|
109
|
+
task_id=prompt.get("task_id", spec.task_id), instruction=prompt["instruction"],
|
|
110
|
+
task_instruction=prompt["task_instruction"], round_instruction=prompt["round_instruction"],
|
|
111
|
+
round_index=prompt["round_index"], robot_id=robot_id or prompt.get("robot_id"),
|
|
112
|
+
episode_index=episode_index,
|
|
113
|
+
scenario_id=getattr(scenario, "scenario_id", None) or prompt.get("scenario_id"),
|
|
114
|
+
success_condition=prompt.get("success_condition", ""),
|
|
115
|
+
constraints=tuple(prompt.get("constraints", ())), public_context=prompt.get("public_context", {}),
|
|
116
|
+
)
|