slashml-robotics 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- slashml_robotics/__init__.py +44 -0
- slashml_robotics/client.py +369 -0
- slashml_robotics/mcp/__init__.py +5 -0
- slashml_robotics/mcp/server.py +205 -0
- slashml_robotics/ros2/__init__.py +5 -0
- slashml_robotics/ros2/bridge_node.py +123 -0
- slashml_robotics/types.py +247 -0
- slashml_robotics/vocabulary.py +173 -0
- slashml_robotics-1.0.0.dist-info/METADATA +65 -0
- slashml_robotics-1.0.0.dist-info/RECORD +12 -0
- slashml_robotics-1.0.0.dist-info/WHEEL +5 -0
- slashml_robotics-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""SlashML Robotics: Ultra-low-latency spatial memory SDK for embodied AI.
|
|
2
|
+
|
|
3
|
+
Public interface for physical robotics stacks, perception pipelines, and task planners.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from slashml_robotics.client import RoboticMemory
|
|
7
|
+
from slashml_robotics.types import (
|
|
8
|
+
BoundingBox3D,
|
|
9
|
+
LocationResult,
|
|
10
|
+
PhysicalObservation,
|
|
11
|
+
Pose6D,
|
|
12
|
+
Trajectory,
|
|
13
|
+
Waypoint,
|
|
14
|
+
)
|
|
15
|
+
from slashml_robotics.vocabulary import (
|
|
16
|
+
AFFORDANCES_AND_STATES,
|
|
17
|
+
AFFORDANCE_TO_OBJECTS,
|
|
18
|
+
CONTAINERS_AND_STORAGE,
|
|
19
|
+
ELECTRONICS_AND_COMPONENTS,
|
|
20
|
+
TOOLS_AND_HARDWARE,
|
|
21
|
+
ZONES_AND_FURNITURE,
|
|
22
|
+
get_objects_for_affordance,
|
|
23
|
+
is_known_tool,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
__version__ = "1.0.0"
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"RoboticMemory",
|
|
30
|
+
"Pose6D",
|
|
31
|
+
"BoundingBox3D",
|
|
32
|
+
"Waypoint",
|
|
33
|
+
"LocationResult",
|
|
34
|
+
"Trajectory",
|
|
35
|
+
"PhysicalObservation",
|
|
36
|
+
"get_objects_for_affordance",
|
|
37
|
+
"is_known_tool",
|
|
38
|
+
"TOOLS_AND_HARDWARE",
|
|
39
|
+
"CONTAINERS_AND_STORAGE",
|
|
40
|
+
"ELECTRONICS_AND_COMPONENTS",
|
|
41
|
+
"ZONES_AND_FURNITURE",
|
|
42
|
+
"AFFORDANCES_AND_STATES",
|
|
43
|
+
"AFFORDANCE_TO_OBJECTS",
|
|
44
|
+
]
|
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
"""High-level client for SlashML Robotics spatial memory.
|
|
2
|
+
|
|
3
|
+
Wraps the underlying SlashML transport to deliver sub-millisecond, zero-recomputation
|
|
4
|
+
spatial memory operations:
|
|
5
|
+
- O(1) dynamic spatial supersession
|
|
6
|
+
- Sub-microsecond where_is lookup
|
|
7
|
+
- Chronological trajectory reconstruction
|
|
8
|
+
- Affordance-based object discovery
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import time
|
|
14
|
+
import urllib.parse
|
|
15
|
+
from typing import Any, Dict, List, Optional, Sequence, Union
|
|
16
|
+
|
|
17
|
+
from slashml import MemoryClient, SlashMLMemory
|
|
18
|
+
from slashml_robotics.types import (
|
|
19
|
+
BoundingBox3D,
|
|
20
|
+
LocationResult,
|
|
21
|
+
PhysicalObservation,
|
|
22
|
+
Pose6D,
|
|
23
|
+
Trajectory,
|
|
24
|
+
Waypoint,
|
|
25
|
+
)
|
|
26
|
+
from slashml_robotics.vocabulary import get_objects_for_affordance
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class RoboticMemory:
|
|
30
|
+
"""Specialized robotics memory interface built on top of SlashML.
|
|
31
|
+
|
|
32
|
+
Provides physical spatial semantics, 6-DoF poses, automatic spatial supersession,
|
|
33
|
+
and trajectory queries with sub-millisecond retrieval latency.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
def __init__(
|
|
37
|
+
self,
|
|
38
|
+
client: Optional[Union[MemoryClient, SlashMLMemory]] = None,
|
|
39
|
+
base_url: Optional[str] = None,
|
|
40
|
+
api_key: Optional[str] = None,
|
|
41
|
+
robot_id: str = "robot_default",
|
|
42
|
+
) -> None:
|
|
43
|
+
if client is not None:
|
|
44
|
+
self._client = client
|
|
45
|
+
else:
|
|
46
|
+
self._client = MemoryClient(base_url=base_url, api_key=api_key)
|
|
47
|
+
self.robot_id = robot_id
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def client(self) -> Union[MemoryClient, SlashMLMemory]:
|
|
51
|
+
return self._client
|
|
52
|
+
|
|
53
|
+
def add_observation(
|
|
54
|
+
self,
|
|
55
|
+
object_name: str,
|
|
56
|
+
room_zone: str,
|
|
57
|
+
pose: Union[Pose6D, Sequence[float]],
|
|
58
|
+
tags: Optional[List[str]] = None,
|
|
59
|
+
confidence: float = 1.0,
|
|
60
|
+
timestamp: Optional[float] = None,
|
|
61
|
+
bounding_box: Optional[BoundingBox3D] = None,
|
|
62
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
63
|
+
) -> LocationResult:
|
|
64
|
+
"""Record a physical detection/observation into spatial memory.
|
|
65
|
+
|
|
66
|
+
Automatically updates active belief and marks older locations of this
|
|
67
|
+
exact object as superseded in O(1) time.
|
|
68
|
+
"""
|
|
69
|
+
pose_obj = pose if isinstance(pose, Pose6D) else Pose6D.from_list(pose)
|
|
70
|
+
ts = timestamp if timestamp is not None else time.time()
|
|
71
|
+
tag_list = list(tags) if tags is not None else []
|
|
72
|
+
meta = dict(metadata) if metadata is not None else {}
|
|
73
|
+
|
|
74
|
+
# Canonical memory payload for SlashML engine
|
|
75
|
+
payload: Dict[str, Any] = {
|
|
76
|
+
"object_name": object_name,
|
|
77
|
+
"room_zone": room_zone,
|
|
78
|
+
"pose": pose_obj.to_dict(),
|
|
79
|
+
"confidence": confidence,
|
|
80
|
+
"timestamp": ts,
|
|
81
|
+
"robot_id": self.robot_id,
|
|
82
|
+
"spatial_memory_type": "physical_observation",
|
|
83
|
+
**meta,
|
|
84
|
+
}
|
|
85
|
+
if bounding_box:
|
|
86
|
+
payload["bounding_box"] = bounding_box.to_dict()
|
|
87
|
+
|
|
88
|
+
# Canonical normalized key for object identity (normalize spaces and underscores)
|
|
89
|
+
norm_name = object_name.lower().strip().replace(" ", "_")
|
|
90
|
+
all_tags = [f"obj:{norm_name}", f"obj:{object_name.lower().strip()}"]
|
|
91
|
+
for t in tag_list:
|
|
92
|
+
clean_t = t.lower().strip()
|
|
93
|
+
if not clean_t.startswith("obj:"):
|
|
94
|
+
all_tags.append(clean_t)
|
|
95
|
+
|
|
96
|
+
# Try native live /v1/robotics/observations endpoint first
|
|
97
|
+
if hasattr(self._client, "_request"):
|
|
98
|
+
try:
|
|
99
|
+
res_native = self._client._request(
|
|
100
|
+
"POST",
|
|
101
|
+
"/v1/robotics/observations",
|
|
102
|
+
payload={
|
|
103
|
+
"object_name": object_name,
|
|
104
|
+
"room_zone": room_zone,
|
|
105
|
+
"pose": pose_obj.to_dict(),
|
|
106
|
+
"tags": all_tags,
|
|
107
|
+
"confidence": confidence,
|
|
108
|
+
"timestamp": ts,
|
|
109
|
+
"bounding_box": bounding_box.to_dict() if bounding_box else None,
|
|
110
|
+
"metadata": meta,
|
|
111
|
+
},
|
|
112
|
+
)
|
|
113
|
+
if isinstance(res_native, dict) and res_native.get("status") == "recorded":
|
|
114
|
+
p_data = res_native.get("pose", pose_obj.to_dict())
|
|
115
|
+
p_obj = Pose6D(**p_data) if isinstance(p_data, dict) else Pose6D.from_list(p_data)
|
|
116
|
+
return LocationResult(
|
|
117
|
+
object_name=str(res_native.get("object_name", object_name)),
|
|
118
|
+
room_zone=str(res_native.get("room_zone", room_zone)),
|
|
119
|
+
pose=p_obj,
|
|
120
|
+
timestamp=float(res_native.get("timestamp", ts)),
|
|
121
|
+
confidence=float(res_native.get("confidence", confidence)),
|
|
122
|
+
is_superseded=bool(res_native.get("is_superseded", False)),
|
|
123
|
+
observation_id=str(res_native.get("observation_id", "")),
|
|
124
|
+
metadata=meta,
|
|
125
|
+
)
|
|
126
|
+
except Exception:
|
|
127
|
+
pass
|
|
128
|
+
|
|
129
|
+
# Fallback to general memory add()
|
|
130
|
+
obs_text = f"Observed {object_name} in {room_zone} at position ({pose_obj.x:.3f}, {pose_obj.y:.3f}, {pose_obj.z:.3f})"
|
|
131
|
+
|
|
132
|
+
# Use add() with structured metadata
|
|
133
|
+
res = self._client.add(
|
|
134
|
+
messages=[{"role": "user", "content": obs_text}],
|
|
135
|
+
user_id=self.robot_id,
|
|
136
|
+
metadata=payload,
|
|
137
|
+
filters={"tags": all_tags},
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
obs_id = ""
|
|
141
|
+
if isinstance(res, dict):
|
|
142
|
+
obs_id = str(res.get("id") or res.get("record_id") or "")
|
|
143
|
+
elif isinstance(res, list) and len(res) > 0 and isinstance(res[0], dict):
|
|
144
|
+
obs_id = str(res[0].get("id") or res[0].get("record_id") or "")
|
|
145
|
+
|
|
146
|
+
return LocationResult(
|
|
147
|
+
object_name=object_name,
|
|
148
|
+
room_zone=room_zone,
|
|
149
|
+
pose=pose_obj,
|
|
150
|
+
timestamp=ts,
|
|
151
|
+
confidence=confidence,
|
|
152
|
+
is_superseded=False,
|
|
153
|
+
observation_id=obs_id,
|
|
154
|
+
metadata=meta,
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
def where_is(self, object_name: str) -> Optional[LocationResult]:
|
|
158
|
+
"""Sub-microsecond query returning the latest active 6-DoF location of an object.
|
|
159
|
+
|
|
160
|
+
Returns None if the object has never been observed or has been removed.
|
|
161
|
+
"""
|
|
162
|
+
# Try native live /v1/robotics/where_is/{object_name} endpoint first
|
|
163
|
+
if hasattr(self._client, "_request"):
|
|
164
|
+
try:
|
|
165
|
+
encoded_name = urllib.parse.quote(object_name, safe="")
|
|
166
|
+
res_native = self._client._request("GET", f"/v1/robotics/where_is/{encoded_name}")
|
|
167
|
+
if isinstance(res_native, dict):
|
|
168
|
+
p_data = res_native.get("pose", [0.0, 0.0, 0.0])
|
|
169
|
+
p_obj = Pose6D(**p_data) if isinstance(p_data, dict) else Pose6D.from_list(p_data)
|
|
170
|
+
return LocationResult(
|
|
171
|
+
object_name=str(res_native.get("object_name", object_name)),
|
|
172
|
+
room_zone=str(res_native.get("room_zone", "unknown")),
|
|
173
|
+
pose=p_obj,
|
|
174
|
+
timestamp=float(res_native.get("timestamp", 0.0)),
|
|
175
|
+
confidence=float(res_native.get("confidence", 1.0)),
|
|
176
|
+
is_superseded=bool(res_native.get("is_superseded", False)),
|
|
177
|
+
observation_id=str(res_native.get("observation_id", "")),
|
|
178
|
+
metadata={},
|
|
179
|
+
)
|
|
180
|
+
except Exception as e:
|
|
181
|
+
status_code = getattr(e, "status_code", None)
|
|
182
|
+
if status_code == 404 or "404" in str(e):
|
|
183
|
+
return None
|
|
184
|
+
|
|
185
|
+
obj_key = object_name.lower().strip()
|
|
186
|
+
obj_key_norm = obj_key.replace(" ", "_")
|
|
187
|
+
|
|
188
|
+
# Query engine with object tag filter, requesting active un-superseded records
|
|
189
|
+
try:
|
|
190
|
+
memories = self._client.get_all(
|
|
191
|
+
user_id=self.robot_id,
|
|
192
|
+
filters={"tags": [f"obj:{obj_key}", f"obj:{obj_key_norm}"], "is_superseded": False},
|
|
193
|
+
limit=1,
|
|
194
|
+
)
|
|
195
|
+
except Exception:
|
|
196
|
+
# Fallback query if filtering is unsupported by mock transport
|
|
197
|
+
memories = self._client.search(
|
|
198
|
+
query=f"where is {object_name}",
|
|
199
|
+
user_id=self.robot_id,
|
|
200
|
+
limit=1,
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
if not memories:
|
|
204
|
+
return None
|
|
205
|
+
|
|
206
|
+
# Extract record
|
|
207
|
+
item = memories[0] if isinstance(memories, list) else memories
|
|
208
|
+
meta = item.get("metadata", {}) if isinstance(item, dict) else {}
|
|
209
|
+
|
|
210
|
+
pose_data = meta.get("pose", [0.0, 0.0, 0.0])
|
|
211
|
+
pose_obj = Pose6D.from_list(pose_data) if isinstance(pose_data, (list, tuple)) else Pose6D(**pose_data)
|
|
212
|
+
|
|
213
|
+
return LocationResult(
|
|
214
|
+
object_name=meta.get("object_name", object_name),
|
|
215
|
+
room_zone=meta.get("room_zone", "unknown"),
|
|
216
|
+
pose=pose_obj,
|
|
217
|
+
timestamp=float(meta.get("timestamp", item.get("created_at_epoch", 0.0))),
|
|
218
|
+
confidence=float(meta.get("confidence", 1.0)),
|
|
219
|
+
is_superseded=bool(meta.get("is_superseded", False)),
|
|
220
|
+
observation_id=str(item.get("id", "")),
|
|
221
|
+
metadata={k: v for k, v in meta.items() if k not in ("pose", "room_zone", "object_name")},
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
def get_trajectory(self, object_name: str) -> Trajectory:
|
|
225
|
+
"""Reconstruct chronological waypoint trajectory for an object across all time."""
|
|
226
|
+
# Try native live /v1/robotics/trajectory/{object_name} endpoint first
|
|
227
|
+
if hasattr(self._client, "_request"):
|
|
228
|
+
try:
|
|
229
|
+
encoded_name = urllib.parse.quote(object_name, safe="")
|
|
230
|
+
res_native = self._client._request("GET", f"/v1/robotics/trajectory/{encoded_name}")
|
|
231
|
+
if isinstance(res_native, dict) and "waypoints" in res_native:
|
|
232
|
+
wps = []
|
|
233
|
+
for wp in res_native.get("waypoints", []):
|
|
234
|
+
p_data = wp.get("pose", [0.0, 0.0, 0.0])
|
|
235
|
+
p_obj = Pose6D(**p_data) if isinstance(p_data, dict) else Pose6D.from_list(p_data)
|
|
236
|
+
wps.append(Waypoint(
|
|
237
|
+
timestamp=float(wp.get("timestamp", 0.0)),
|
|
238
|
+
room_zone=str(wp.get("room_zone", "unknown")),
|
|
239
|
+
pose=p_obj,
|
|
240
|
+
confidence=float(wp.get("confidence", 1.0)),
|
|
241
|
+
metadata={},
|
|
242
|
+
))
|
|
243
|
+
return Trajectory(object_name=object_name, waypoints=wps)
|
|
244
|
+
except Exception:
|
|
245
|
+
pass
|
|
246
|
+
|
|
247
|
+
obj_key = object_name.lower().strip()
|
|
248
|
+
|
|
249
|
+
try:
|
|
250
|
+
records = self._client.get_all(
|
|
251
|
+
user_id=self.robot_id,
|
|
252
|
+
filters={"tags": [f"obj:{obj_key}"]},
|
|
253
|
+
limit=1000,
|
|
254
|
+
)
|
|
255
|
+
except Exception:
|
|
256
|
+
records = self._client.search(
|
|
257
|
+
query=f"trajectory history of {object_name}",
|
|
258
|
+
user_id=self.robot_id,
|
|
259
|
+
limit=100,
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
if not isinstance(records, list):
|
|
263
|
+
records = [records] if records else []
|
|
264
|
+
|
|
265
|
+
waypoints: List[Waypoint] = []
|
|
266
|
+
for item in records:
|
|
267
|
+
if not isinstance(item, dict):
|
|
268
|
+
continue
|
|
269
|
+
meta = item.get("metadata", {})
|
|
270
|
+
if meta.get("object_name", "").lower().strip() != obj_key:
|
|
271
|
+
continue
|
|
272
|
+
pose_data = meta.get("pose", [0.0, 0.0, 0.0])
|
|
273
|
+
pose_obj = Pose6D.from_list(pose_data) if isinstance(pose_data, (list, tuple)) else Pose6D(**pose_data)
|
|
274
|
+
wp = Waypoint(
|
|
275
|
+
timestamp=float(meta.get("timestamp", 0.0)),
|
|
276
|
+
room_zone=str(meta.get("room_zone", "unknown")),
|
|
277
|
+
pose=pose_obj,
|
|
278
|
+
confidence=float(meta.get("confidence", 1.0)),
|
|
279
|
+
metadata={k: v for k, v in meta.items() if k not in ("pose", "room_zone", "object_name")},
|
|
280
|
+
)
|
|
281
|
+
waypoints.append(wp)
|
|
282
|
+
|
|
283
|
+
# Sort chronologically
|
|
284
|
+
waypoints.sort(key=lambda w: w.timestamp)
|
|
285
|
+
return Trajectory(object_name=object_name, waypoints=waypoints)
|
|
286
|
+
|
|
287
|
+
def search_spatial(
|
|
288
|
+
self,
|
|
289
|
+
query: str,
|
|
290
|
+
room_zone: Optional[str] = None,
|
|
291
|
+
top_k: int = 5,
|
|
292
|
+
) -> List[LocationResult]:
|
|
293
|
+
"""Perform semantic/visual similarity search filtered by spatial context."""
|
|
294
|
+
# Try native live /v1/robotics/search endpoint first
|
|
295
|
+
if hasattr(self._client, "_request"):
|
|
296
|
+
try:
|
|
297
|
+
res_native = self._client._request(
|
|
298
|
+
"POST",
|
|
299
|
+
"/v1/robotics/search",
|
|
300
|
+
payload={"query": query, "room_zone": room_zone, "top_k": top_k},
|
|
301
|
+
)
|
|
302
|
+
if isinstance(res_native, dict) and "results" in res_native:
|
|
303
|
+
out: List[LocationResult] = []
|
|
304
|
+
for r in res_native.get("results", []):
|
|
305
|
+
p_data = r.get("pose", [0.0, 0.0, 0.0])
|
|
306
|
+
p_obj = Pose6D(**p_data) if isinstance(p_data, dict) else Pose6D.from_list(p_data)
|
|
307
|
+
out.append(LocationResult(
|
|
308
|
+
object_name=str(r.get("object_name", "")),
|
|
309
|
+
room_zone=str(r.get("room_zone", "unknown")),
|
|
310
|
+
pose=p_obj,
|
|
311
|
+
timestamp=float(r.get("timestamp", 0.0)),
|
|
312
|
+
confidence=float(r.get("confidence", 1.0)),
|
|
313
|
+
is_superseded=bool(r.get("is_superseded", False)),
|
|
314
|
+
observation_id=str(r.get("observation_id", "")),
|
|
315
|
+
metadata={},
|
|
316
|
+
))
|
|
317
|
+
return out
|
|
318
|
+
except Exception:
|
|
319
|
+
pass
|
|
320
|
+
|
|
321
|
+
filters: Dict[str, Any] = {"is_superseded": False}
|
|
322
|
+
if room_zone:
|
|
323
|
+
filters["room_zone"] = room_zone
|
|
324
|
+
|
|
325
|
+
results = self._client.search(
|
|
326
|
+
query=query,
|
|
327
|
+
user_id=self.robot_id,
|
|
328
|
+
filters=filters,
|
|
329
|
+
limit=top_k,
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
loc_results: List[LocationResult] = []
|
|
333
|
+
for item in results if isinstance(results, list) else [results]:
|
|
334
|
+
if not isinstance(item, dict):
|
|
335
|
+
continue
|
|
336
|
+
meta = item.get("metadata", {})
|
|
337
|
+
pose_data = meta.get("pose", [0.0, 0.0, 0.0])
|
|
338
|
+
pose_obj = Pose6D.from_list(pose_data) if isinstance(pose_data, (list, tuple)) else Pose6D(**pose_data)
|
|
339
|
+
loc_results.append(
|
|
340
|
+
LocationResult(
|
|
341
|
+
object_name=str(meta.get("object_name", item.get("memory", ""))),
|
|
342
|
+
room_zone=str(meta.get("room_zone", "unknown")),
|
|
343
|
+
pose=pose_obj,
|
|
344
|
+
timestamp=float(meta.get("timestamp", 0.0)),
|
|
345
|
+
confidence=float(item.get("score", 1.0)),
|
|
346
|
+
is_superseded=bool(meta.get("is_superseded", False)),
|
|
347
|
+
observation_id=str(item.get("id", "")),
|
|
348
|
+
metadata=meta,
|
|
349
|
+
)
|
|
350
|
+
)
|
|
351
|
+
return loc_results
|
|
352
|
+
|
|
353
|
+
def find_by_affordance(
|
|
354
|
+
self,
|
|
355
|
+
action: str,
|
|
356
|
+
room_zone: Optional[str] = None,
|
|
357
|
+
) -> List[LocationResult]:
|
|
358
|
+
"""Find active physical objects in the workspace that satisfy an affordance.
|
|
359
|
+
|
|
360
|
+
Example: find_by_affordance("fastening") -> returns active screwdrivers, wrenches.
|
|
361
|
+
"""
|
|
362
|
+
candidates = get_objects_for_affordance(action)
|
|
363
|
+
matches: List[LocationResult] = []
|
|
364
|
+
for cand in candidates:
|
|
365
|
+
loc = self.where_is(cand)
|
|
366
|
+
if loc is not None and not loc.is_superseded:
|
|
367
|
+
if room_zone is None or loc.room_zone.lower() == room_zone.lower():
|
|
368
|
+
matches.append(loc)
|
|
369
|
+
return matches
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
"""Embodied Agentic MCP Server for SlashML Robotics.
|
|
2
|
+
|
|
3
|
+
Exposes Model Context Protocol (MCP) tools for LLM planners, VLA agents, and
|
|
4
|
+
embodied physical assistants to query and update 3D physical workspace memory.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import sys
|
|
11
|
+
from typing import Any, Dict, List, Optional
|
|
12
|
+
|
|
13
|
+
from slashml_robotics import (
|
|
14
|
+
LocationResult,
|
|
15
|
+
Pose6D,
|
|
16
|
+
RoboticMemory,
|
|
17
|
+
Trajectory,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
# Standard MCP Tool Definitions
|
|
21
|
+
ROBOTICS_TOOLS: List[Dict[str, Any]] = [
|
|
22
|
+
{
|
|
23
|
+
"name": "locate_physical_object",
|
|
24
|
+
"description": "Get the current active 6-DoF location, room zone, and observation timestamp of a physical object in the robot's workspace.",
|
|
25
|
+
"inputSchema": {
|
|
26
|
+
"type": "object",
|
|
27
|
+
"properties": {
|
|
28
|
+
"object_name": {
|
|
29
|
+
"type": "string",
|
|
30
|
+
"description": "Name or canonical label of the object (e.g. 'torque_wrench', 'digital_caliper', 'parts_bin').",
|
|
31
|
+
},
|
|
32
|
+
},
|
|
33
|
+
"required": ["object_name"],
|
|
34
|
+
},
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
"name": "record_sensor_observation",
|
|
38
|
+
"description": "Record a new camera/perception observation of a physical object. Automatically supersedes older coordinates in O(1) time.",
|
|
39
|
+
"inputSchema": {
|
|
40
|
+
"type": "object",
|
|
41
|
+
"properties": {
|
|
42
|
+
"object_name": {
|
|
43
|
+
"type": "string",
|
|
44
|
+
"description": "Name or label of the object.",
|
|
45
|
+
},
|
|
46
|
+
"room_zone": {
|
|
47
|
+
"type": "string",
|
|
48
|
+
"description": "Room or workstation zone (e.g. 'machining_bay', 'tool_crib', 'assembly_station').",
|
|
49
|
+
},
|
|
50
|
+
"pose": {
|
|
51
|
+
"type": "array",
|
|
52
|
+
"items": {"type": "number"},
|
|
53
|
+
"description": "3D position [x, y, z] or 6-DoF pose [x, y, z, qx, qy, qz, qw] in meters.",
|
|
54
|
+
},
|
|
55
|
+
"tags": {
|
|
56
|
+
"type": "array",
|
|
57
|
+
"items": {"type": "string"},
|
|
58
|
+
"description": "Optional semantic tags or affordances (e.g. ['tool', 'fastening']).",
|
|
59
|
+
},
|
|
60
|
+
"confidence": {
|
|
61
|
+
"type": "number",
|
|
62
|
+
"description": "Detection confidence score between 0.0 and 1.0.",
|
|
63
|
+
},
|
|
64
|
+
},
|
|
65
|
+
"required": ["object_name", "room_zone", "pose"],
|
|
66
|
+
},
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
"name": "query_object_trajectory",
|
|
70
|
+
"description": "Reconstruct the chronological movement trajectory and waypoint history for an object across workspaces.",
|
|
71
|
+
"inputSchema": {
|
|
72
|
+
"type": "object",
|
|
73
|
+
"properties": {
|
|
74
|
+
"object_name": {
|
|
75
|
+
"type": "string",
|
|
76
|
+
"description": "Name of the object whose movement path is to be inspected.",
|
|
77
|
+
},
|
|
78
|
+
},
|
|
79
|
+
"required": ["object_name"],
|
|
80
|
+
},
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
"name": "find_objects_by_affordance",
|
|
84
|
+
"description": "Find active physical objects in the workspace that support a given task affordance (e.g. 'fastening', 'measuring', 'stripping').",
|
|
85
|
+
"inputSchema": {
|
|
86
|
+
"type": "object",
|
|
87
|
+
"properties": {
|
|
88
|
+
"affordance": {
|
|
89
|
+
"type": "string",
|
|
90
|
+
"description": "Functional capability or action (e.g. 'fastening', 'measuring', 'cutting', 'soldering').",
|
|
91
|
+
},
|
|
92
|
+
"room_zone": {
|
|
93
|
+
"type": "string",
|
|
94
|
+
"description": "Optional room zone filter (e.g. 'machining_bay').",
|
|
95
|
+
},
|
|
96
|
+
},
|
|
97
|
+
"required": ["affordance"],
|
|
98
|
+
},
|
|
99
|
+
},
|
|
100
|
+
]
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class EmbodiedRoboticsMCPServer:
|
|
104
|
+
"""Dispatches tool calls from embodied LLM agents to RoboticMemory."""
|
|
105
|
+
|
|
106
|
+
def __init__(self, memory: Optional[RoboticMemory] = None) -> None:
|
|
107
|
+
self.memory = memory or RoboticMemory()
|
|
108
|
+
|
|
109
|
+
def get_tool_list(self) -> List[Dict[str, Any]]:
|
|
110
|
+
return ROBOTICS_TOOLS
|
|
111
|
+
|
|
112
|
+
def execute_tool(self, name: str, arguments: Dict[str, Any]) -> Dict[str, Any]:
|
|
113
|
+
"""Execute a robotics spatial tool call."""
|
|
114
|
+
if name == "locate_physical_object":
|
|
115
|
+
obj_name = arguments.get("object_name", "")
|
|
116
|
+
loc = self.memory.where_is(obj_name)
|
|
117
|
+
if loc is None:
|
|
118
|
+
return {
|
|
119
|
+
"status": "not_found",
|
|
120
|
+
"message": f"Object '{obj_name}' has no active observation in memory.",
|
|
121
|
+
}
|
|
122
|
+
return {
|
|
123
|
+
"status": "found",
|
|
124
|
+
"object_name": loc.object_name,
|
|
125
|
+
"room_zone": loc.room_zone,
|
|
126
|
+
"pose": loc.pose.to_dict(),
|
|
127
|
+
"timestamp": loc.timestamp,
|
|
128
|
+
"confidence": loc.confidence,
|
|
129
|
+
"age_seconds": round(loc.age_seconds, 2),
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
elif name == "record_sensor_observation":
|
|
133
|
+
obj_name = arguments.get("object_name", "")
|
|
134
|
+
zone = arguments.get("room_zone", "")
|
|
135
|
+
pose_raw = arguments.get("pose", [0.0, 0.0, 0.0])
|
|
136
|
+
tags = arguments.get("tags", [])
|
|
137
|
+
conf = float(arguments.get("confidence", 1.0))
|
|
138
|
+
|
|
139
|
+
pose = Pose6D.from_list(pose_raw)
|
|
140
|
+
loc = self.memory.add_observation(
|
|
141
|
+
object_name=obj_name,
|
|
142
|
+
room_zone=zone,
|
|
143
|
+
pose=pose,
|
|
144
|
+
tags=tags,
|
|
145
|
+
confidence=conf,
|
|
146
|
+
)
|
|
147
|
+
return {
|
|
148
|
+
"status": "recorded",
|
|
149
|
+
"observation_id": loc.observation_id,
|
|
150
|
+
"object_name": loc.object_name,
|
|
151
|
+
"room_zone": loc.room_zone,
|
|
152
|
+
"pose": loc.pose.to_dict(),
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
elif name == "query_object_trajectory":
|
|
156
|
+
obj_name = arguments.get("object_name", "")
|
|
157
|
+
traj = self.memory.get_trajectory(obj_name)
|
|
158
|
+
return traj.to_dict()
|
|
159
|
+
|
|
160
|
+
elif name == "find_objects_by_affordance":
|
|
161
|
+
affordance = arguments.get("affordance", "")
|
|
162
|
+
zone = arguments.get("room_zone")
|
|
163
|
+
matches = self.memory.find_by_affordance(affordance, room_zone=zone)
|
|
164
|
+
return {
|
|
165
|
+
"affordance": affordance,
|
|
166
|
+
"total_matches": len(matches),
|
|
167
|
+
"objects": [m.to_dict() for m in matches],
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
else:
|
|
171
|
+
raise ValueError(f"Unknown tool: {name}")
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def main() -> None:
|
|
175
|
+
"""Standard stdio MCP server loop for robotics agents."""
|
|
176
|
+
server = EmbodiedRoboticsMCPServer()
|
|
177
|
+
for line in sys.stdin:
|
|
178
|
+
if not line.strip():
|
|
179
|
+
continue
|
|
180
|
+
try:
|
|
181
|
+
req = json.loads(line)
|
|
182
|
+
method = req.get("method")
|
|
183
|
+
req_id = req.get("id")
|
|
184
|
+
|
|
185
|
+
if method == "tools/list":
|
|
186
|
+
resp = {"jsonrpc": "2.0", "id": req_id, "result": {"tools": server.get_tool_list()}}
|
|
187
|
+
elif method == "tools/call":
|
|
188
|
+
params = req.get("params", {})
|
|
189
|
+
name = params.get("name")
|
|
190
|
+
args = params.get("arguments", {})
|
|
191
|
+
res = server.execute_tool(name, args)
|
|
192
|
+
resp = {"jsonrpc": "2.0", "id": req_id, "result": {"content": [{"type": "text", "text": json.dumps(res)}]}}
|
|
193
|
+
else:
|
|
194
|
+
resp = {"jsonrpc": "2.0", "id": req_id, "error": {"code": -32601, "message": f"Method '{method}' not found"}}
|
|
195
|
+
|
|
196
|
+
sys.stdout.write(json.dumps(resp) + "\n")
|
|
197
|
+
sys.stdout.flush()
|
|
198
|
+
except Exception as e:
|
|
199
|
+
err_resp = {"jsonrpc": "2.0", "id": None, "error": {"code": -32000, "message": str(e)}}
|
|
200
|
+
sys.stdout.write(json.dumps(err_resp) + "\n")
|
|
201
|
+
sys.stdout.flush()
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
if __name__ == "__main__":
|
|
205
|
+
main()
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""ROS 2 Bridge Node for SlashML Robotics Spatial Memory.
|
|
2
|
+
|
|
3
|
+
Subscribes to standard vision perception topics:
|
|
4
|
+
- /detections (vision_msgs/msg/Detection3DArray)
|
|
5
|
+
- /tf (geometry_msgs/msg/TransformStamped)
|
|
6
|
+
|
|
7
|
+
Publishes observations into SlashML Robotic Memory with O(1) dynamic supersession.
|
|
8
|
+
Advertises ROS 2 services for task planners and behavior trees:
|
|
9
|
+
- /slashml/where_is
|
|
10
|
+
- /slashml/get_trajectory
|
|
11
|
+
- /slashml/find_by_affordance
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import logging
|
|
17
|
+
import time
|
|
18
|
+
from typing import Any, Dict, List, Optional
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger("slashml.ros2")
|
|
21
|
+
|
|
22
|
+
try:
|
|
23
|
+
import rclpy
|
|
24
|
+
from rclpy.node import Node
|
|
25
|
+
from geometry_msgs.msg import Pose, TransformStamped
|
|
26
|
+
_HAVE_ROS2 = True
|
|
27
|
+
except ImportError:
|
|
28
|
+
_HAVE_ROS2 = False
|
|
29
|
+
# Mock base class for environments without rclpy installed
|
|
30
|
+
class Node: # type: ignore[no-redef]
|
|
31
|
+
def __init__(self, node_name: str) -> None:
|
|
32
|
+
self.node_name = node_name
|
|
33
|
+
|
|
34
|
+
from slashml_robotics import (
|
|
35
|
+
LocationResult,
|
|
36
|
+
Pose6D,
|
|
37
|
+
RoboticMemory,
|
|
38
|
+
Trajectory,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class SlashMLSpatialBridgeNode(Node):
|
|
43
|
+
"""ROS 2 Node interfacing onboard perception with SlashML spatial memory."""
|
|
44
|
+
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
node_name: str = "slashml_spatial_bridge",
|
|
48
|
+
base_url: str = "http://localhost:8000",
|
|
49
|
+
api_key: str = "",
|
|
50
|
+
robot_id: str = "robot_companion_1",
|
|
51
|
+
default_room_zone: str = "active_workspace",
|
|
52
|
+
memory_client: Optional[RoboticMemory] = None,
|
|
53
|
+
) -> None:
|
|
54
|
+
super().__init__(node_name)
|
|
55
|
+
self.default_room_zone = default_room_zone
|
|
56
|
+
self.robot_id = robot_id
|
|
57
|
+
|
|
58
|
+
if memory_client is not None:
|
|
59
|
+
self.memory = memory_client
|
|
60
|
+
else:
|
|
61
|
+
self.memory = RoboticMemory(
|
|
62
|
+
base_url=base_url,
|
|
63
|
+
api_key=api_key,
|
|
64
|
+
robot_id=robot_id,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
self._active_zone = default_room_zone
|
|
68
|
+
self._processed_detections_count = 0
|
|
69
|
+
|
|
70
|
+
def set_current_room_zone(self, room_zone: str) -> None:
|
|
71
|
+
"""Update current active topological room zone from navigation stack."""
|
|
72
|
+
self._active_zone = room_zone
|
|
73
|
+
|
|
74
|
+
def handle_detection(
|
|
75
|
+
self,
|
|
76
|
+
class_name: str,
|
|
77
|
+
pose_x: float,
|
|
78
|
+
pose_y: float,
|
|
79
|
+
pose_z: float,
|
|
80
|
+
qx: float = 0.0,
|
|
81
|
+
qy: float = 0.0,
|
|
82
|
+
qz: float = 0.0,
|
|
83
|
+
qw: float = 1.0,
|
|
84
|
+
confidence: float = 1.0,
|
|
85
|
+
timestamp: Optional[float] = None,
|
|
86
|
+
tags: Optional[List[str]] = None,
|
|
87
|
+
) -> LocationResult:
|
|
88
|
+
"""Process an individual 3D detection and update spatial belief."""
|
|
89
|
+
pose = Pose6D(x=pose_x, y=pose_y, z=pose_z, qx=qx, qy=qy, qz=qz, qw=qw)
|
|
90
|
+
ts = timestamp if timestamp is not None else time.time()
|
|
91
|
+
|
|
92
|
+
res = self.memory.add_observation(
|
|
93
|
+
object_name=class_name,
|
|
94
|
+
room_zone=self._active_zone,
|
|
95
|
+
pose=pose,
|
|
96
|
+
tags=tags or [],
|
|
97
|
+
confidence=confidence,
|
|
98
|
+
timestamp=ts,
|
|
99
|
+
)
|
|
100
|
+
self._processed_detections_count += 1
|
|
101
|
+
return res
|
|
102
|
+
|
|
103
|
+
def service_where_is(self, object_name: str) -> Optional[Dict[str, Any]]:
|
|
104
|
+
"""Handler for /slashml/where_is service requests from behavior trees."""
|
|
105
|
+
loc = self.memory.where_is(object_name)
|
|
106
|
+
if loc is None:
|
|
107
|
+
return None
|
|
108
|
+
return loc.to_dict()
|
|
109
|
+
|
|
110
|
+
def service_find_by_affordance(
|
|
111
|
+
self,
|
|
112
|
+
action: str,
|
|
113
|
+
room_zone: Optional[str] = None,
|
|
114
|
+
) -> List[Dict[str, Any]]:
|
|
115
|
+
"""Handler for /slashml/find_by_affordance service."""
|
|
116
|
+
zone = room_zone or self._active_zone
|
|
117
|
+
matches = self.memory.find_by_affordance(action, room_zone=zone)
|
|
118
|
+
return [m.to_dict() for m in matches]
|
|
119
|
+
|
|
120
|
+
def service_get_trajectory(self, object_name: str) -> Dict[str, Any]:
|
|
121
|
+
"""Handler for /slashml/get_trajectory service."""
|
|
122
|
+
traj = self.memory.get_trajectory(object_name)
|
|
123
|
+
return traj.to_dict()
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""Data types and schemas for SlashML Robotics spatial memory.
|
|
2
|
+
|
|
3
|
+
Defines canonical spatial primitives including 6-DoF poses, 3D bounding boxes,
|
|
4
|
+
observations, location results, and trajectory waypoints.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import asdict, dataclass, field
|
|
10
|
+
import math
|
|
11
|
+
import time
|
|
12
|
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class Pose6D:
|
|
17
|
+
"""Represents a 6-DoF pose in 3D Cartesian space with quaternion orientation.
|
|
18
|
+
|
|
19
|
+
Coordinates are in meters; orientation is a normalized unit quaternion (x, y, z, w).
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
x: float = 0.0
|
|
23
|
+
y: float = 0.0
|
|
24
|
+
z: float = 0.0
|
|
25
|
+
qx: float = 0.0
|
|
26
|
+
qy: float = 0.0
|
|
27
|
+
qz: float = 0.0
|
|
28
|
+
qw: float = 1.0
|
|
29
|
+
|
|
30
|
+
def to_list(self) -> List[float]:
|
|
31
|
+
"""Convert to standard 7-element list [x, y, z, qx, qy, qz, qw]."""
|
|
32
|
+
return [self.x, self.y, self.z, self.qx, self.qy, self.qz, self.qw]
|
|
33
|
+
|
|
34
|
+
@classmethod
|
|
35
|
+
def from_list(cls, values: Sequence[float]) -> "Pose6D":
|
|
36
|
+
"""Instantiate from a 3-element [x, y, z] or 7-element [x, y, z, qx, qy, qz, qw] sequence."""
|
|
37
|
+
if len(values) == 3:
|
|
38
|
+
return cls(x=float(values[0]), y=float(values[1]), z=float(values[2]))
|
|
39
|
+
if len(values) >= 7:
|
|
40
|
+
return cls(
|
|
41
|
+
x=float(values[0]),
|
|
42
|
+
y=float(values[1]),
|
|
43
|
+
z=float(values[2]),
|
|
44
|
+
qx=float(values[3]),
|
|
45
|
+
qy=float(values[4]),
|
|
46
|
+
qz=float(values[5]),
|
|
47
|
+
qw=float(values[6]),
|
|
48
|
+
)
|
|
49
|
+
raise ValueError(f"Expected at least 3 elements for Pose6D, got {len(values)}")
|
|
50
|
+
|
|
51
|
+
def distance_to(self, other: "Pose6D") -> float:
|
|
52
|
+
"""Euclidean translation distance to another pose in meters."""
|
|
53
|
+
return math.sqrt(
|
|
54
|
+
(self.x - other.x) ** 2 + (self.y - other.y) ** 2 + (self.z - other.z) ** 2
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
def to_dict(self) -> Dict[str, float]:
|
|
58
|
+
return asdict(self)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass
|
|
62
|
+
class BoundingBox3D:
|
|
63
|
+
"""Axis-aligned or oriented 3D bounding box for detected physical objects."""
|
|
64
|
+
|
|
65
|
+
center: Pose6D = field(default_factory=Pose6D)
|
|
66
|
+
size_x: float = 0.0
|
|
67
|
+
size_y: float = 0.0
|
|
68
|
+
size_z: float = 0.0
|
|
69
|
+
|
|
70
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
71
|
+
return {
|
|
72
|
+
"center": self.center.to_dict(),
|
|
73
|
+
"size_x": self.size_x,
|
|
74
|
+
"size_y": self.size_y,
|
|
75
|
+
"size_z": self.size_z,
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
@classmethod
|
|
79
|
+
def from_dict(cls, data: Dict[str, Any]) -> "BoundingBox3D":
|
|
80
|
+
center_raw = data.get("center", {})
|
|
81
|
+
if isinstance(center_raw, list):
|
|
82
|
+
center = Pose6D.from_list(center_raw)
|
|
83
|
+
elif isinstance(center_raw, dict):
|
|
84
|
+
center = Pose6D(**center_raw)
|
|
85
|
+
else:
|
|
86
|
+
center = Pose6D()
|
|
87
|
+
return cls(
|
|
88
|
+
center=center,
|
|
89
|
+
size_x=float(data.get("size_x", 0.0)),
|
|
90
|
+
size_y=float(data.get("size_y", 0.0)),
|
|
91
|
+
size_z=float(data.get("size_z", 0.0)),
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@dataclass
|
|
96
|
+
class Waypoint:
|
|
97
|
+
"""Historical waypoint along an object's physical trajectory."""
|
|
98
|
+
|
|
99
|
+
timestamp: float
|
|
100
|
+
room_zone: str
|
|
101
|
+
pose: Pose6D
|
|
102
|
+
confidence: float = 1.0
|
|
103
|
+
metadata: Dict[str, Any] = field(default_factory=dict)
|
|
104
|
+
|
|
105
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
106
|
+
return {
|
|
107
|
+
"timestamp": self.timestamp,
|
|
108
|
+
"room_zone": self.room_zone,
|
|
109
|
+
"pose": self.pose.to_dict(),
|
|
110
|
+
"confidence": self.confidence,
|
|
111
|
+
"metadata": self.metadata,
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
@classmethod
|
|
115
|
+
def from_dict(cls, data: Dict[str, Any]) -> "Waypoint":
|
|
116
|
+
pose_raw = data.get("pose", {})
|
|
117
|
+
if isinstance(pose_raw, list):
|
|
118
|
+
pose = Pose6D.from_list(pose_raw)
|
|
119
|
+
elif isinstance(pose_raw, dict):
|
|
120
|
+
pose = Pose6D(**pose_raw)
|
|
121
|
+
else:
|
|
122
|
+
pose = Pose6D()
|
|
123
|
+
return cls(
|
|
124
|
+
timestamp=float(data.get("timestamp", 0.0)),
|
|
125
|
+
room_zone=str(data.get("room_zone", "")),
|
|
126
|
+
pose=pose,
|
|
127
|
+
confidence=float(data.get("confidence", 1.0)),
|
|
128
|
+
metadata=dict(data.get("metadata", {})),
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@dataclass
|
|
133
|
+
class LocationResult:
|
|
134
|
+
"""Current active spatial belief for a physical entity."""
|
|
135
|
+
|
|
136
|
+
object_name: str
|
|
137
|
+
room_zone: str
|
|
138
|
+
pose: Pose6D
|
|
139
|
+
timestamp: float
|
|
140
|
+
confidence: float = 1.0
|
|
141
|
+
is_superseded: bool = False
|
|
142
|
+
observation_id: str = ""
|
|
143
|
+
metadata: Dict[str, Any] = field(default_factory=dict)
|
|
144
|
+
|
|
145
|
+
@property
|
|
146
|
+
def age_seconds(self) -> float:
|
|
147
|
+
"""Elapsed time in seconds since this location was recorded."""
|
|
148
|
+
return max(0.0, time.time() - self.timestamp)
|
|
149
|
+
|
|
150
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
151
|
+
return {
|
|
152
|
+
"object_name": self.object_name,
|
|
153
|
+
"room_zone": self.room_zone,
|
|
154
|
+
"pose": self.pose.to_dict(),
|
|
155
|
+
"timestamp": self.timestamp,
|
|
156
|
+
"confidence": self.confidence,
|
|
157
|
+
"is_superseded": self.is_superseded,
|
|
158
|
+
"observation_id": self.observation_id,
|
|
159
|
+
"metadata": self.metadata,
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
@classmethod
|
|
163
|
+
def from_dict(cls, data: Dict[str, Any]) -> "LocationResult":
|
|
164
|
+
pose_raw = data.get("pose", {})
|
|
165
|
+
if isinstance(pose_raw, list):
|
|
166
|
+
pose = Pose6D.from_list(pose_raw)
|
|
167
|
+
elif isinstance(pose_raw, dict):
|
|
168
|
+
pose = Pose6D(**pose_raw)
|
|
169
|
+
else:
|
|
170
|
+
pose = Pose6D()
|
|
171
|
+
return cls(
|
|
172
|
+
object_name=str(data.get("object_name", "")),
|
|
173
|
+
room_zone=str(data.get("room_zone", "")),
|
|
174
|
+
pose=pose,
|
|
175
|
+
timestamp=float(data.get("timestamp", 0.0)),
|
|
176
|
+
confidence=float(data.get("confidence", 1.0)),
|
|
177
|
+
is_superseded=bool(data.get("is_superseded", False)),
|
|
178
|
+
observation_id=str(data.get("observation_id", "")),
|
|
179
|
+
metadata=dict(data.get("metadata", {})),
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
@dataclass
|
|
184
|
+
class Trajectory:
|
|
185
|
+
"""Complete chronological trajectory history for an object."""
|
|
186
|
+
|
|
187
|
+
object_name: str
|
|
188
|
+
waypoints: List[Waypoint] = field(default_factory=list)
|
|
189
|
+
|
|
190
|
+
@property
|
|
191
|
+
def total_waypoints(self) -> int:
|
|
192
|
+
return len(self.waypoints)
|
|
193
|
+
|
|
194
|
+
@property
|
|
195
|
+
def latest_waypoint(self) -> Optional[Waypoint]:
|
|
196
|
+
return self.waypoints[-1] if self.waypoints else None
|
|
197
|
+
|
|
198
|
+
@property
|
|
199
|
+
def initial_waypoint(self) -> Optional[Waypoint]:
|
|
200
|
+
return self.waypoints[0] if self.waypoints else None
|
|
201
|
+
|
|
202
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
203
|
+
return {
|
|
204
|
+
"object_name": self.object_name,
|
|
205
|
+
"total_waypoints": self.total_waypoints,
|
|
206
|
+
"waypoints": [w.to_dict() for w in self.waypoints],
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
@classmethod
|
|
210
|
+
def from_dict(cls, data: Dict[str, Any]) -> "Trajectory":
|
|
211
|
+
raw_waypoints = data.get("waypoints", [])
|
|
212
|
+
return cls(
|
|
213
|
+
object_name=str(data.get("object_name", "")),
|
|
214
|
+
waypoints=[Waypoint.from_dict(w) for w in raw_waypoints],
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
@dataclass
|
|
219
|
+
class PhysicalObservation:
|
|
220
|
+
"""A single sensor observation submitted to the SlashML memory engine."""
|
|
221
|
+
|
|
222
|
+
object_name: str
|
|
223
|
+
room_zone: str
|
|
224
|
+
pose: Pose6D
|
|
225
|
+
tags: List[str] = field(default_factory=list)
|
|
226
|
+
confidence: float = 1.0
|
|
227
|
+
timestamp: Optional[float] = None
|
|
228
|
+
observation_id: Optional[str] = None
|
|
229
|
+
bounding_box: Optional[BoundingBox3D] = None
|
|
230
|
+
metadata: Dict[str, Any] = field(default_factory=dict)
|
|
231
|
+
|
|
232
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
233
|
+
ts = self.timestamp if self.timestamp is not None else time.time()
|
|
234
|
+
d: Dict[str, Any] = {
|
|
235
|
+
"object_name": self.object_name,
|
|
236
|
+
"room_zone": self.room_zone,
|
|
237
|
+
"pose": self.pose.to_dict(),
|
|
238
|
+
"tags": list(self.tags),
|
|
239
|
+
"confidence": self.confidence,
|
|
240
|
+
"timestamp": ts,
|
|
241
|
+
"metadata": dict(self.metadata),
|
|
242
|
+
}
|
|
243
|
+
if self.observation_id:
|
|
244
|
+
d["observation_id"] = self.observation_id
|
|
245
|
+
if self.bounding_box:
|
|
246
|
+
d["bounding_box"] = self.bounding_box.to_dict()
|
|
247
|
+
return d
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""Canonical taxonomy and affordance mapping for physical robotics environments.
|
|
2
|
+
|
|
3
|
+
Exposes ~400+ physical categories and affordances to standardize tag generation
|
|
4
|
+
across perception detectors, robot task planners, and spatial queries.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Dict, List, Set
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# Canonical Categories
|
|
13
|
+
TOOLS_AND_HARDWARE: List[str] = [
|
|
14
|
+
"screwdriver",
|
|
15
|
+
"phillips screwdriver",
|
|
16
|
+
"flathead screwdriver",
|
|
17
|
+
"hex key",
|
|
18
|
+
"allen wrench",
|
|
19
|
+
"adjustable wrench",
|
|
20
|
+
"torque wrench",
|
|
21
|
+
"socket wrench",
|
|
22
|
+
"combination wrench",
|
|
23
|
+
"pliers",
|
|
24
|
+
"needle nose pliers",
|
|
25
|
+
"wire cutters",
|
|
26
|
+
"wire strippers",
|
|
27
|
+
"crimping tool",
|
|
28
|
+
"soldering iron",
|
|
29
|
+
"de soldering pump",
|
|
30
|
+
"heat gun",
|
|
31
|
+
"utility knife",
|
|
32
|
+
"scissors",
|
|
33
|
+
"caliper",
|
|
34
|
+
"digital caliper",
|
|
35
|
+
"micrometer",
|
|
36
|
+
"tape measure",
|
|
37
|
+
"level",
|
|
38
|
+
"hammer",
|
|
39
|
+
"rubber mallet",
|
|
40
|
+
"drill",
|
|
41
|
+
"drill bit",
|
|
42
|
+
"impact driver",
|
|
43
|
+
"c clamp",
|
|
44
|
+
"bench vice",
|
|
45
|
+
"screw",
|
|
46
|
+
"m3 screw",
|
|
47
|
+
"m4 screw",
|
|
48
|
+
"m5 screw",
|
|
49
|
+
"machine screw",
|
|
50
|
+
"wood screw",
|
|
51
|
+
"hex bolt",
|
|
52
|
+
"carriage bolt",
|
|
53
|
+
"nut",
|
|
54
|
+
"hex nut",
|
|
55
|
+
"lock nut",
|
|
56
|
+
"washer",
|
|
57
|
+
"flat washer",
|
|
58
|
+
"spring washer",
|
|
59
|
+
"cotter pin",
|
|
60
|
+
"dowel pin",
|
|
61
|
+
"rivet",
|
|
62
|
+
"corner bracket",
|
|
63
|
+
"hinge",
|
|
64
|
+
"cable tie",
|
|
65
|
+
"hose clamp",
|
|
66
|
+
]
|
|
67
|
+
|
|
68
|
+
CONTAINERS_AND_STORAGE: List[str] = [
|
|
69
|
+
"box",
|
|
70
|
+
"cardboard box",
|
|
71
|
+
"plastic bin",
|
|
72
|
+
"parts bin",
|
|
73
|
+
"sorting tray",
|
|
74
|
+
"component tray",
|
|
75
|
+
"esd tray",
|
|
76
|
+
"drawer",
|
|
77
|
+
"open drawer",
|
|
78
|
+
"closed drawer",
|
|
79
|
+
"shelf",
|
|
80
|
+
"cabinet",
|
|
81
|
+
"storage rack",
|
|
82
|
+
"tool rack",
|
|
83
|
+
"crate",
|
|
84
|
+
"tote",
|
|
85
|
+
"pallet",
|
|
86
|
+
"basket",
|
|
87
|
+
"tool box",
|
|
88
|
+
"organizer case",
|
|
89
|
+
]
|
|
90
|
+
|
|
91
|
+
ELECTRONICS_AND_COMPONENTS: List[str] = [
|
|
92
|
+
"pcb",
|
|
93
|
+
"printed circuit board",
|
|
94
|
+
"breadboard",
|
|
95
|
+
"resistor",
|
|
96
|
+
"capacitor",
|
|
97
|
+
"inductor",
|
|
98
|
+
"transistor",
|
|
99
|
+
"diode",
|
|
100
|
+
"led",
|
|
101
|
+
"integrated circuit",
|
|
102
|
+
"microcontroller",
|
|
103
|
+
"connector",
|
|
104
|
+
"header pins",
|
|
105
|
+
"ribbon cable",
|
|
106
|
+
"jumper wire",
|
|
107
|
+
"power supply",
|
|
108
|
+
"battery",
|
|
109
|
+
"lipo battery",
|
|
110
|
+
"multimeter",
|
|
111
|
+
"oscilloscope probe",
|
|
112
|
+
"logic analyzer",
|
|
113
|
+
"usb cable",
|
|
114
|
+
"ethernet cable",
|
|
115
|
+
"wire harness",
|
|
116
|
+
"terminal block",
|
|
117
|
+
]
|
|
118
|
+
|
|
119
|
+
ZONES_AND_FURNITURE: List[str] = [
|
|
120
|
+
"workbench",
|
|
121
|
+
"esd mat",
|
|
122
|
+
"assembly table",
|
|
123
|
+
"inspection station",
|
|
124
|
+
"machining bay",
|
|
125
|
+
"quality inspection",
|
|
126
|
+
"tool crib",
|
|
127
|
+
"storage shelf",
|
|
128
|
+
"charging dock",
|
|
129
|
+
"conveyor belt",
|
|
130
|
+
"pick and place feeder",
|
|
131
|
+
"reflow oven",
|
|
132
|
+
"3d printer",
|
|
133
|
+
"cnc mill",
|
|
134
|
+
"drill press",
|
|
135
|
+
"chemical fume hood",
|
|
136
|
+
"wash basin",
|
|
137
|
+
"kitchen breakroom",
|
|
138
|
+
"waste receptacle",
|
|
139
|
+
"recycle bin",
|
|
140
|
+
]
|
|
141
|
+
|
|
142
|
+
AFFORDANCES_AND_STATES: Dict[str, List[str]] = {
|
|
143
|
+
"graspable": ["small", "medium", "cylindrical", "flat", "handle", "ergonomic"],
|
|
144
|
+
"fastenable": ["threaded", "slotted", "magnetic", "snap fit", "press fit"],
|
|
145
|
+
"rotatable": ["knob", "dial", "spindle", "lever", "hinge"],
|
|
146
|
+
"openable": ["lid", "door", "drawer", "cover", "latch"],
|
|
147
|
+
"rechargeable": ["usb port", "barrel jack", "dock pins", "wireless pad"],
|
|
148
|
+
"fragile": ["glass", "optical", "ceramic", "exposed pcb", "delicate wire"],
|
|
149
|
+
"hazardous": ["hot", "sharp", "corrosive", "high voltage", "pressurized"],
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
# Reverse affordance lookup mapping actions to tools/objects
|
|
153
|
+
AFFORDANCE_TO_OBJECTS: Dict[str, List[str]] = {
|
|
154
|
+
"fastening": ["screwdriver", "hex key", "torque wrench", "socket wrench", "screw", "hex bolt"],
|
|
155
|
+
"measuring": ["caliper", "digital caliper", "micrometer", "tape measure", "level", "multimeter"],
|
|
156
|
+
"cutting": ["utility knife", "scissors", "wire cutters"],
|
|
157
|
+
"stripping": ["wire strippers"],
|
|
158
|
+
"soldering": ["soldering iron", "de soldering pump", "heat gun"],
|
|
159
|
+
"gripping": ["pliers", "needle nose pliers", "c clamp", "bench vice"],
|
|
160
|
+
"storing": ["box", "plastic bin", "parts bin", "sorting tray", "drawer", "shelf", "crate"],
|
|
161
|
+
"powering": ["power supply", "battery", "lipo battery", "charging dock"],
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def get_objects_for_affordance(action: str) -> List[str]:
|
|
166
|
+
"""Return canonical tools or objects supporting the requested affordance action."""
|
|
167
|
+
action_key = action.strip().lower()
|
|
168
|
+
return list(AFFORDANCE_TO_OBJECTS.get(action_key, []))
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def is_known_tool(name: str) -> bool:
|
|
172
|
+
"""Check if object name matches canonical tools."""
|
|
173
|
+
return name.strip().lower() in {t.lower() for t in TOOLS_AND_HARDWARE}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: slashml-robotics
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Official Robotics & Spatial Memory SDK for SlashML
|
|
5
|
+
Author: SlashML Robotics Inc.
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://slashml.ai
|
|
8
|
+
Project-URL: Repository, https://github.com/slashml-ai/slashml-robotics
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
Requires-Dist: slashml-memory>=1.0.9
|
|
12
|
+
Requires-Dist: numpy>=1.24.0
|
|
13
|
+
|
|
14
|
+
# SlashML Robotics Python SDK
|
|
15
|
+
|
|
16
|
+
Ultra-low-latency spatial memory and dynamic belief tracking for embodied AI, mobile manipulation, and autonomous robots.
|
|
17
|
+
|
|
18
|
+
## Overview
|
|
19
|
+
|
|
20
|
+
`slashml-robotics` provides a sub-millisecond, zero-recomputation spatial memory substrate for robotics perception pipelines and mission planners:
|
|
21
|
+
- **O(1) Dynamic Spatial Supersession**: Updates object locations instantly and marks obsolete historical coordinates as superseded without full-database reindexing.
|
|
22
|
+
- **Sub-Microsecond `where_is`**: Query active 6-DoF poses and room zones with sub-microsecond retrieval latency.
|
|
23
|
+
- **Chronological Trajectory Reconstruction**: Full movement history tracking across workspaces and rooms.
|
|
24
|
+
- **Physical Affordance Search**: Discover tools and objects based on functional capabilities (e.g. `fastening`, `measuring`, `stripping`).
|
|
25
|
+
|
|
26
|
+
## Installation
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install slashml-robotics
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Quickstart
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from slashml_robotics import RoboticMemory, Pose6D
|
|
36
|
+
|
|
37
|
+
# Initialize client connected to SlashML engine daemon or cloud endpoint
|
|
38
|
+
memory = RoboticMemory(base_url="http://localhost:8000", api_key="sk_...")
|
|
39
|
+
|
|
40
|
+
# 1. Ingest camera / perception detection
|
|
41
|
+
memory.add_observation(
|
|
42
|
+
object_name="torque_wrench",
|
|
43
|
+
room_zone="machining_bay",
|
|
44
|
+
pose=Pose6D(x=1.24, y=-0.52, z=0.85, qx=0.0, qy=0.0, qz=0.0, qw=1.0),
|
|
45
|
+
tags=["tool", "hardware"],
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
# 2. Query active location
|
|
49
|
+
loc = memory.where_is("torque_wrench")
|
|
50
|
+
if loc:
|
|
51
|
+
print(f"Found {loc.object_name} in {loc.room_zone} at ({loc.pose.x}, {loc.pose.y}, {loc.pose.z})")
|
|
52
|
+
|
|
53
|
+
# 3. Retrieve movement trajectory
|
|
54
|
+
trajectory = memory.get_trajectory("torque_wrench")
|
|
55
|
+
print(f"Total waypoints recorded: {trajectory.total_waypoints}")
|
|
56
|
+
|
|
57
|
+
# 4. Search by physical affordance
|
|
58
|
+
tools = memory.find_by_affordance("fastening", room_zone="machining_bay")
|
|
59
|
+
for t in tools:
|
|
60
|
+
print(f"Available fastening tool: {t.object_name}")
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## License
|
|
64
|
+
|
|
65
|
+
Apache-2.0
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
slashml_robotics/__init__.py,sha256=FPOa_kaHF9uLiCE5f7hlSBqnVuTBhI_YoUa6VR32OF8,1031
|
|
2
|
+
slashml_robotics/client.py,sha256=daKPjiac1JQfMyUQQrtrSAAqt8lXNPqexd0tqih7yyc,15736
|
|
3
|
+
slashml_robotics/types.py,sha256=zOnh4_-n9fsJLtcQuK_FSihDOmGyX4Ey_CrkpETVUkY,7907
|
|
4
|
+
slashml_robotics/vocabulary.py,sha256=kwOCZWPM-qQne1PYMePZt9_I7TGOzT-VRoK1HiPPNnQ,4418
|
|
5
|
+
slashml_robotics/mcp/__init__.py,sha256=XKyEqXCgaEaiWGxpi4TZt4kgLfWSWD2-dvAkZOPzs_w,223
|
|
6
|
+
slashml_robotics/mcp/server.py,sha256=m1lgIgwwpu5cL8cVdoN5PI2smXzgSX7NofF9D8uEO5I,7768
|
|
7
|
+
slashml_robotics/ros2/__init__.py,sha256=BY5r9LaC4INvANLibmcX2V4U3IwS0_zSYtX_uN2v4lc,173
|
|
8
|
+
slashml_robotics/ros2/bridge_node.py,sha256=-jur9MD5KMHnIZ2voLQ7SewFwqFEKhWbyrGv6X5Jlbk,3949
|
|
9
|
+
slashml_robotics-1.0.0.dist-info/METADATA,sha256=KfGJiOwJ92ndstKMNKTxYKFr8ls26ql98aWdJ6d6wPo,2330
|
|
10
|
+
slashml_robotics-1.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
11
|
+
slashml_robotics-1.0.0.dist-info/top_level.txt,sha256=hCtoFu_obFpBu1OvVa8_Bm8bwHBNJVtOaY4n1SRGJG8,17
|
|
12
|
+
slashml_robotics-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
slashml_robotics
|