prototwin-gymnasium 0.1.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prototwin_gymnasium-0.1.8/LICENSE.txt +17 -0
- prototwin_gymnasium-0.1.8/MANIFEST.in +3 -0
- prototwin_gymnasium-0.1.8/PKG-INFO +407 -0
- prototwin_gymnasium-0.1.8/README.md +385 -0
- prototwin_gymnasium-0.1.8/prototwin_gymnasium/__init__.py +1 -0
- prototwin_gymnasium-0.1.8/prototwin_gymnasium/env.py +292 -0
- prototwin_gymnasium-0.1.8/prototwin_gymnasium.egg-info/PKG-INFO +407 -0
- prototwin_gymnasium-0.1.8/prototwin_gymnasium.egg-info/SOURCES.txt +13 -0
- prototwin_gymnasium-0.1.8/prototwin_gymnasium.egg-info/dependency_links.txt +1 -0
- prototwin_gymnasium-0.1.8/prototwin_gymnasium.egg-info/requires.txt +6 -0
- prototwin_gymnasium-0.1.8/prototwin_gymnasium.egg-info/top_level.txt +1 -0
- prototwin_gymnasium-0.1.8/requirements.txt +6 -0
- prototwin_gymnasium-0.1.8/setup.cfg +9 -0
- prototwin_gymnasium-0.1.8/setup.py +28 -0
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
Copyright (c) 2018 YOUR NAME
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
4
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
5
|
+
in the Software without restriction, including without limitation the rights
|
|
6
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
7
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
8
|
+
furnished to do so, subject to the following conditions:
|
|
9
|
+
The above copyright notice and this permission notice shall be included in all
|
|
10
|
+
copies or substantial portions of the Software.
|
|
11
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
12
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
13
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
14
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
15
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
16
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
17
|
+
SOFTWARE.
|
|
@@ -0,0 +1,407 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: prototwin_gymnasium
|
|
3
|
+
Version: 0.1.8
|
|
4
|
+
Summary: The official base Gymnasium environment for ProtoTwin Connect.
|
|
5
|
+
Home-page: https://prototwin.com
|
|
6
|
+
Author: ProtoTwin
|
|
7
|
+
License: MIT
|
|
8
|
+
Keywords: Industrial Simulation,Simulation,Physics,Machine Learning,Reinforcement Learning,Robotics,Gymnasium
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE.txt
|
|
16
|
+
Requires-Dist: gymnasium
|
|
17
|
+
Requires-Dist: prototwin
|
|
18
|
+
Requires-Dist: asyncio
|
|
19
|
+
Requires-Dist: nest_asyncio
|
|
20
|
+
Requires-Dist: numpy
|
|
21
|
+
Requires-Dist: stable_baselines3
|
|
22
|
+
|
|
23
|
+
# ProtoTwin Gymnasium Environment
|
|
24
|
+
|
|
25
|
+
This package provides a base environment for [Gymnasium](https://gymnasium.farama.org/index.html), to be used for reinforcement learning.
|
|
26
|
+
|
|
27
|
+
## Introduction
|
|
28
|
+
|
|
29
|
+
ProtoTwin Connect allows external applications to issue commands for loading models, stepping the simulation and reading/writing signals. This package provides a base environment for Gymnasium, a library for reinforcement learning. ProtoTwin Connect is a drop-in replacement for existing physics libraries like PyBullet and MuJoCo. The advantage provided by ProtoTwin Connect is that you don't need to programatically create your robot/machine or start with an existing URDF file. Instead, you can import your CAD into ProtoTwin and, with a few clicks of the mouse, define rigid bodies, collision geometry, friction materials, joints and motors.
|
|
30
|
+
|
|
31
|
+
### Signals
|
|
32
|
+
|
|
33
|
+
Signals represent I/O for components defined in ProtoTwin. The [prototwin package](https://pypi.org/project/prototwin/) provides a client for starting and connecting to an instance of ProtoTwin Connect. Using this client you can issue commands to load a model, step the simulation forwards in time, read signal values and write signal values. Some examples of signals include:
|
|
34
|
+
|
|
35
|
+
* The current simulation time
|
|
36
|
+
* The target position for a motor
|
|
37
|
+
* The current velocity of motor
|
|
38
|
+
* The current force/torque applied by a motor
|
|
39
|
+
* The state of a volumetric sensor (blocked/cleared)
|
|
40
|
+
* The distance measured by a distance sensor
|
|
41
|
+
* The accelerations measured by an accelerometer
|
|
42
|
+
|
|
43
|
+
Signals are either readable or writable. For example, the current simulation time is readable whilst the target position for a motor is writable. Writable signals can also be read, but readable signals cannot be written.
|
|
44
|
+
|
|
45
|
+
#### Types
|
|
46
|
+
|
|
47
|
+
Signals are strongly typed. The following value types are supported:
|
|
48
|
+
|
|
49
|
+
* Boolean
|
|
50
|
+
* Uint8
|
|
51
|
+
* Uint16
|
|
52
|
+
* Uint32
|
|
53
|
+
* Int8
|
|
54
|
+
* Int16
|
|
55
|
+
* Int32
|
|
56
|
+
* Float
|
|
57
|
+
* Double
|
|
58
|
+
|
|
59
|
+
You can find the signals provided by each component inside ProtoTwin under the I/O dropdown menu. The I/O window lists the name, address and type of each signal along with its access (readable/writable). Python does not natively support small integral types. The client will automatically clamp values to the range of the integral type. For example, attempting to set a signal of type Uint8 to the value 1000 will cause the value to be clamped at 255.
|
|
60
|
+
|
|
61
|
+
#### Custom Signals
|
|
62
|
+
|
|
63
|
+
Many of the components built into ProtoTwin provide their own set of signals. However, it is also possible to create your own components with their own set of signals. This is done my creating a scripted component inside of ProtoTwin and assigning that component to one or more entities. The example below demonstrates a simple custom component that generates a sinusoidal wave and provides a readable signal for the amplitude of the wave. The value of this signal can be read at runtime through the ProtoTwin Connect python client.
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
import { Component, type Entity, IO, DoubleSignal, Access } from "prototwin";
|
|
67
|
+
|
|
68
|
+
export class SineWaveGeneratorIO extends IO {
|
|
69
|
+
public wave: DoubleSignal;
|
|
70
|
+
|
|
71
|
+
public constructor(component: SineWaveGenerator) {
|
|
72
|
+
super(component);
|
|
73
|
+
this.wave = new DoubleSignal(0, Access.Readable);
|
|
74
|
+
this.assign(); // Assign addresses to signals so that they can be accessed by connected clients (e.g. Python)
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export class SineWaveGenerator extends Component {
|
|
79
|
+
#io: SineWaveGeneratorIO;
|
|
80
|
+
|
|
81
|
+
public override get io(): SineWaveGeneratorIO {
|
|
82
|
+
return this.#io;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
constructor(entity: Entity) {
|
|
86
|
+
super(entity);
|
|
87
|
+
this.#io = new SineWaveGeneratorIO(this);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
public override update(dt: number) {
|
|
91
|
+
this.#io.wave.value = Math.sin(this.entity.world.time);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Example
|
|
97
|
+
|
|
98
|
+
This example demonstrates training an [inverted pendulum to swing up and balance](https://www.youtube.com/watch?v=W9wx2ZqYVJA).
|
|
99
|
+
Note that this example requires [ProtoTwin Connect](https://prototwin.com) to be installed on your local machine.
|
|
100
|
+
|
|
101
|
+
### Non-Vectorized
|
|
102
|
+
|
|
103
|
+
```
|
|
104
|
+
# STEP 1: Import dependencies
|
|
105
|
+
import prototwin_gymnasium
|
|
106
|
+
import prototwin
|
|
107
|
+
import asyncio
|
|
108
|
+
import os
|
|
109
|
+
import math
|
|
110
|
+
import time
|
|
111
|
+
import keyboard
|
|
112
|
+
import numpy as np
|
|
113
|
+
import torch as th
|
|
114
|
+
from typing import Tuple
|
|
115
|
+
from gymnasium import spaces
|
|
116
|
+
from stable_baselines3 import PPO
|
|
117
|
+
from stable_baselines3.common.policies import BasePolicy
|
|
118
|
+
from stable_baselines3.common.callbacks import CheckpointCallback
|
|
119
|
+
|
|
120
|
+
# STEP 2: Define signal addresses (obtain these values from ProtoTwin)
|
|
121
|
+
address_time = 0
|
|
122
|
+
address_cart_target_velocity = 3
|
|
123
|
+
address_cart_position = 5
|
|
124
|
+
address_cart_velocity = 6
|
|
125
|
+
address_cart_force = 7
|
|
126
|
+
address_pole_angle = 12
|
|
127
|
+
address_pole_angular_velocity = 13
|
|
128
|
+
|
|
129
|
+
# STEP 3: Create your environment by extending the base environment
|
|
130
|
+
class CartPoleEnv(prototwin_gymnasium.Env):
|
|
131
|
+
def __init__(self, client: prototwin.Client) -> None:
|
|
132
|
+
super().__init__(client)
|
|
133
|
+
self.x_threshold = 0.65 # Maximum cart distance
|
|
134
|
+
|
|
135
|
+
# The action space contains only the cart's target velocity
|
|
136
|
+
action_high = np.array([1.0], dtype=np.float32)
|
|
137
|
+
self.action_space = spaces.Box(-action_high, action_high, dtype=np.float32)
|
|
138
|
+
|
|
139
|
+
# The observation space contains:
|
|
140
|
+
# 0. A measure of the cart's distance from the center, where 0 is at the center and +/-1 is at the limit.
|
|
141
|
+
# 1. A measure of the angular distance of the pole from the upright position, where 0 is at the upright position and 1 is at the down position.
|
|
142
|
+
# 2. The cart's current velocity (m/s).
|
|
143
|
+
# 3. The pole's angular velocity (rad/s).
|
|
144
|
+
observation_high = np.array([1, 1, np.finfo(np.float32).max, np.finfo(np.float32).max], dtype=np.float32)
|
|
145
|
+
self.observation_space = spaces.Box(-observation_high, observation_high, dtype=np.float32)
|
|
146
|
+
|
|
147
|
+
def reward(self, obs):
|
|
148
|
+
distance = 1 - math.fabs(obs[0]) # How close the cart is to the center
|
|
149
|
+
angle = 1 - math.fabs(obs[1]) # How close the pole is to the upright position
|
|
150
|
+
force = math.fabs(self.get(address_cart_force)) # How much force is being applied to drive the cart's motor
|
|
151
|
+
return (angle * angle) * 0.8 + (distance * distance) * 0.2 - force * 0.001
|
|
152
|
+
|
|
153
|
+
def reset(self, seed = None):
|
|
154
|
+
super().reset(seed=seed)
|
|
155
|
+
return np.array([0, 0])
|
|
156
|
+
|
|
157
|
+
def step(self, action):
|
|
158
|
+
self.set(address_cart_target_velocity, action) # Apply action by setting the cart's target velocity
|
|
159
|
+
super().step() # Step the simulation forwards by one time-step
|
|
160
|
+
time = self.get(address_time) # Read the current simulation time
|
|
161
|
+
cart_position = self.get(address_cart_position) # Read the current cart position
|
|
162
|
+
cart_velocity = self.get(address_cart_velocity) # Read the current cart velocity
|
|
163
|
+
pole_angle = self.get(address_pole_angle) # Read the current pole angle
|
|
164
|
+
pole_angular_velocity = self.get(address_pole_angular_velocity) # Read the current pole angular velocity
|
|
165
|
+
pole_angular_distance = math.atan2(math.sin(math.pi - pole_angle), math.cos(math.pi - pole_angle)) # Calculate angular distance from upright position
|
|
166
|
+
obs = np.array([cart_position / self.x_threshold, pole_angular_distance / math.pi, cart_velocity, pole_angular_velocity]) # Set observation space
|
|
167
|
+
reward = self.reward(obs) # Calculate reward
|
|
168
|
+
done = abs(obs[0]) > 1 # Terminate if cart goes beyond limits
|
|
169
|
+
truncated = time > 20 # Truncate after 20 seconds
|
|
170
|
+
return obs, reward, done, truncated, {}
|
|
171
|
+
|
|
172
|
+
# STEP 4: Setup the training session
|
|
173
|
+
async def train():
|
|
174
|
+
# Start ProtoTwin Connect
|
|
175
|
+
client = await prototwin.start()
|
|
176
|
+
|
|
177
|
+
# Load the ProtoTwin model
|
|
178
|
+
filepath = os.path.join(os.path.dirname(__file__), "CartPole.ptm")
|
|
179
|
+
await client.load(filepath)
|
|
180
|
+
|
|
181
|
+
# Create the environment
|
|
182
|
+
env = CartPoleEnv(client)
|
|
183
|
+
|
|
184
|
+
# Define the ML model
|
|
185
|
+
model = PPO("MlpPolicy", env, verbose=1, tensorboard_log="./tensorboard/")
|
|
186
|
+
|
|
187
|
+
# Create callback to regularly save the model
|
|
188
|
+
checkpoint_callback = CheckpointCallback(save_freq=save_freq, save_path="./logs/checkpoints/", name_prefix="checkpoint", save_replay_buffer=True, save_vecnormalize=True)
|
|
189
|
+
|
|
190
|
+
# Start learning!
|
|
191
|
+
model.learn(total_timesteps=2_000_000, callback=checkpoint_callback)
|
|
192
|
+
|
|
193
|
+
# STEP 5: Setup the evaluation session
|
|
194
|
+
async def evaluate():
|
|
195
|
+
# Start ProtoTwin Connect
|
|
196
|
+
client = await prototwin.start()
|
|
197
|
+
|
|
198
|
+
# Load the ProtoTwin model
|
|
199
|
+
filepath = os.path.join(os.path.dirname(__file__), "CartPole.ptm")
|
|
200
|
+
await client.load(filepath)
|
|
201
|
+
|
|
202
|
+
# Create the environment
|
|
203
|
+
env = CartPoleEnv(client)
|
|
204
|
+
|
|
205
|
+
# Load the trained ML model
|
|
206
|
+
model = PPO.load("logs/best_model/best_model", env)
|
|
207
|
+
|
|
208
|
+
# Run simulation at real-time speed
|
|
209
|
+
while True:
|
|
210
|
+
env.reset()
|
|
211
|
+
done = False
|
|
212
|
+
obs = [0, 0, 0, 0]
|
|
213
|
+
start_wall_time = time.perf_counter()
|
|
214
|
+
while not done:
|
|
215
|
+
action, _ = model.predict(obs)
|
|
216
|
+
obs, reward, done, truncated, info = env.step(action)
|
|
217
|
+
if keyboard.is_pressed("r"):
|
|
218
|
+
break
|
|
219
|
+
current_wall_time = time.perf_counter()
|
|
220
|
+
elapsed_wall_time = current_wall_time - start_wall_time
|
|
221
|
+
elapsed_sim_time = client.get(address_time)
|
|
222
|
+
sleep_time = elapsed_sim_time - elapsed_wall_time
|
|
223
|
+
if sleep_time > 0:
|
|
224
|
+
await asyncio.sleep(sleep_time)
|
|
225
|
+
|
|
226
|
+
asyncio.run(train())
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
### Vectorized
|
|
230
|
+
|
|
231
|
+
```
|
|
232
|
+
# STEP 1: Import dependencies
|
|
233
|
+
import asyncio
|
|
234
|
+
import stable_baselines3.ppo
|
|
235
|
+
import stable_baselines3.ppo.ppo
|
|
236
|
+
import torch
|
|
237
|
+
import os
|
|
238
|
+
import gymnasium
|
|
239
|
+
import numpy as np
|
|
240
|
+
import math
|
|
241
|
+
import prototwin
|
|
242
|
+
import stable_baselines3
|
|
243
|
+
import stable_baselines3.common
|
|
244
|
+
import stable_baselines3.common.vec_env
|
|
245
|
+
import stable_baselines3.common.vec_env.base_vec_env
|
|
246
|
+
from stable_baselines3 import PPO
|
|
247
|
+
from stable_baselines3.common.vec_env import VecMonitor
|
|
248
|
+
from stable_baselines3.common.callbacks import CheckpointCallback
|
|
249
|
+
from prototwin_gymnasium import VecEnvInstance, VecEnv
|
|
250
|
+
|
|
251
|
+
# STEP 2: Define signal addresses (obtain these values from ProtoTwin)
|
|
252
|
+
address_time = 0
|
|
253
|
+
address_cart_target_velocity = 3
|
|
254
|
+
address_cart_position = 5
|
|
255
|
+
address_cart_velocity = 6
|
|
256
|
+
address_cart_force = 7
|
|
257
|
+
address_pole_angle = 12
|
|
258
|
+
address_pole_angular_velocity = 13
|
|
259
|
+
|
|
260
|
+
# STEP 3: Create your vectorized instance environment by extending the base environment
|
|
261
|
+
class CartPoleEnv(VecEnvInstance):
|
|
262
|
+
def __init__(self, client: prototwin.Client, instance: int) -> None:
|
|
263
|
+
super().__init__(client, instance)
|
|
264
|
+
self.x_threshold = 0.65 # Maximum cart distance
|
|
265
|
+
|
|
266
|
+
# The action space contains only the cart's target velocity
|
|
267
|
+
action_high = np.array([1.0], dtype=np.float32)
|
|
268
|
+
self.action_space = gymnasium.spaces.Box(-action_high, action_high, dtype=np.float32)
|
|
269
|
+
|
|
270
|
+
# The observation space contains:
|
|
271
|
+
# 0. A measure of the cart's distance from the center, where 0 is at the center and +/-1 is at the limit.
|
|
272
|
+
# 1. A measure of the angular distance of the pole from the upright position, where 0 is at the upright position and 1 is at the down position.
|
|
273
|
+
# 2. The cart's current velocity (m/s).
|
|
274
|
+
# 3. The pole's angular velocity (rad/s).
|
|
275
|
+
observation_high = np.array([1, 1, np.finfo(np.float32).max, np.finfo(np.float32).max], dtype=np.float32)
|
|
276
|
+
self.observation_space = gymnasium.spaces.Box(-observation_high, observation_high, dtype=np.float32)
|
|
277
|
+
|
|
278
|
+
def reward(self, obs):
|
|
279
|
+
distance = 1 - math.fabs(obs[0]) # How close the cart is to the center
|
|
280
|
+
angle = 1 - math.fabs(obs[1]) # How close the pole is to the upright position
|
|
281
|
+
force = math.fabs(self.get(address_cart_force)) # How much force is being applied to drive the cart's motor
|
|
282
|
+
return (angle * angle) * 0.8 + (distance * distance) * 0.2 - force * 0.001
|
|
283
|
+
|
|
284
|
+
def observations(self):
|
|
285
|
+
cart_position = self.get(address_cart_position) # Read the current cart position
|
|
286
|
+
cart_velocity = self.get(address_cart_velocity) # Read the current cart velocity
|
|
287
|
+
pole_angle = self.get(address_pole_angle) # Read the current pole angle
|
|
288
|
+
pole_angular_velocity = self.get(address_pole_angular_velocity) # Read the current pole angular velocity
|
|
289
|
+
pole_angular_distance = math.atan2(math.sin(math.pi - pole_angle), math.cos(math.pi - pole_angle)) # Calculate angular distance from upright position
|
|
290
|
+
return np.array([cart_position / self.x_threshold, pole_angular_distance / math.pi, cart_velocity, pole_angular_velocity])
|
|
291
|
+
|
|
292
|
+
def reset(self, seed = None):
|
|
293
|
+
super().reset(seed=seed)
|
|
294
|
+
return self.observations(), {}
|
|
295
|
+
|
|
296
|
+
def apply(self, action):
|
|
297
|
+
self.set(address_cart_target_velocity, action[0]) # Apply action by setting the cart's target velocity
|
|
298
|
+
|
|
299
|
+
def step(self):
|
|
300
|
+
obs = self.observations()
|
|
301
|
+
reward = self.reward(obs) # Calculate reward
|
|
302
|
+
done = abs(obs[0]) > 1 # Terminate if cart goes beyond limits
|
|
303
|
+
truncated = self.time > 20 # Truncate after 20 seconds
|
|
304
|
+
return obs, reward, done, truncated, {}
|
|
305
|
+
|
|
306
|
+
# STEP 4: Setup the training session
|
|
307
|
+
async def main():
|
|
308
|
+
# Start ProtoTwin Connect
|
|
309
|
+
client = await prototwin.start()
|
|
310
|
+
|
|
311
|
+
# Load the ProtoTwin model
|
|
312
|
+
filepath = os.path.join(os.path.dirname(__file__), "CartPole.ptm")
|
|
313
|
+
await client.load(filepath)
|
|
314
|
+
|
|
315
|
+
# Create the vectorized environment
|
|
316
|
+
observation_high = np.array([1, 1, 1, np.finfo(np.float32).max], dtype=np.float32)
|
|
317
|
+
observation_space = gymnasium.spaces.Box(-observation_high, observation_high, dtype=np.float32)
|
|
318
|
+
action_high = np.array([1.0], dtype=np.float32)
|
|
319
|
+
action_space = gymnasium.spaces.Box(-action_high, action_high, dtype=np.float32)
|
|
320
|
+
source_entity_name = "Main"
|
|
321
|
+
instance_count = 64
|
|
322
|
+
env = VecEnv(CartPoleEnv, client, source_entity_name, instance_count, observation_space, action_space)
|
|
323
|
+
monitored = VecMonitor(env) # Monitor the training progress
|
|
324
|
+
|
|
325
|
+
# Create callback to regularly save the model
|
|
326
|
+
save_freq = 10000 # Number of timesteps per instance
|
|
327
|
+
checkpoint_callback = CheckpointCallback(save_freq=save_freq, save_path="./logs/checkpoints/", name_prefix="checkpoint", save_replay_buffer=True, save_vecnormalize=True)
|
|
328
|
+
|
|
329
|
+
# Start learning!
|
|
330
|
+
model = PPO(stable_baselines3.ppo.MlpPolicy, monitored, device=torch.cuda.current_device(), verbose=1, batch_size=4096, n_steps=1000, learning_rate=0.0003, tensorboard_log="./tensorboard/")
|
|
331
|
+
model.learn(total_timesteps=10_000_000, callback=checkpoint_callback)
|
|
332
|
+
|
|
333
|
+
asyncio.run(main())
|
|
334
|
+
```
|
|
335
|
+
|
|
336
|
+
## Exporting to ONNX
|
|
337
|
+
|
|
338
|
+
It is possible to export trained models to the ONNX format. This can be used to embed trained agents into ProtoTwin models for inferencing. Please refer to the [Stable Baselines exporting documentation](https://stable-baselines3.readthedocs.io/en/master/guide/export.html) for further details. The example provided below shows how to export the trained Cart Pole model to ONNX.
|
|
339
|
+
|
|
340
|
+
```
|
|
341
|
+
# Export to ONNX for embedding into ProtoTwin models using ONNX Runtime Web
|
|
342
|
+
def export():
|
|
343
|
+
class OnnxableSB3Policy(th.nn.Module):
|
|
344
|
+
def __init__(self, policy: BasePolicy):
|
|
345
|
+
super().__init__()
|
|
346
|
+
self.policy = policy
|
|
347
|
+
|
|
348
|
+
def forward(self, observation: th.Tensor) -> Tuple[th.Tensor, th.Tensor, th.Tensor]:
|
|
349
|
+
return self.policy(observation, deterministic=True)
|
|
350
|
+
|
|
351
|
+
# Load the trained ML model
|
|
352
|
+
model = PPO.load("logs/best_model/best_model", device="cpu")
|
|
353
|
+
|
|
354
|
+
# Create the Onnx policy
|
|
355
|
+
onnx_policy = OnnxableSB3Policy(model.policy)
|
|
356
|
+
|
|
357
|
+
observation_size = model.observation_space.shape
|
|
358
|
+
dummy_input = th.randn(1, *observation_size)
|
|
359
|
+
th.onnx.export(onnx_policy, dummy_input, "CartPole.onnx", opset_version=17, input_names=["input"], output_names=["output"])
|
|
360
|
+
```
|
|
361
|
+
|
|
362
|
+
## Inference in ProtoTwin
|
|
363
|
+
|
|
364
|
+
It is possible to embed trained agents into ProtoTwin models. To do this, you must create a scripted component that loads the ONNX model, feeds observations into the model and finally applies the output actions. Note that this example assumes that the ONNX file has been included into the model by dragging the file into the script editor's file explorer. Alternatively, the ONNX file can be loaded from a URL.
|
|
365
|
+
|
|
366
|
+
```
|
|
367
|
+
import { type Entity, type Handle, InferenceComponent, MotorComponent, Util } from "prototwin";
|
|
368
|
+
|
|
369
|
+
export class CartPole extends InferenceComponent {
|
|
370
|
+
public cartMotor: Handle<MotorComponent>;
|
|
371
|
+
public poleMotor: Handle<MotorComponent>;
|
|
372
|
+
|
|
373
|
+
constructor(entity: Entity) {
|
|
374
|
+
super(entity);
|
|
375
|
+
this.cartMotor = this.handle(MotorComponent);
|
|
376
|
+
this.poleMotor = this.handle(MotorComponent);
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
public override async initializeAsync() {
|
|
380
|
+
// Load the ONNX model from the local filesystem.
|
|
381
|
+
this.loadModelFromFile("CartPole.onnx", 4, [-1], [1]);
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
public override async updateAsync() {
|
|
385
|
+
const cartMotor = this.cartMotor.value;
|
|
386
|
+
const poleMotor = this.poleMotor.value;
|
|
387
|
+
const observations = this.observations;
|
|
388
|
+
if (cartMotor === null || poleMotor === null || observations === null) { return; }
|
|
389
|
+
|
|
390
|
+
// Populate observation array
|
|
391
|
+
const cartPosition = cartMotor.currentPosition;
|
|
392
|
+
const cartVelocity = cartMotor.currentVelocity;
|
|
393
|
+
const poleAngularDistance = Util.signedAngularDifference(poleMotor.currentPosition, Math.PI);
|
|
394
|
+
const poleAngularVelocity = poleMotor.currentVelocity;
|
|
395
|
+
observations[0] = cartPosition / 0.65;
|
|
396
|
+
observations[1] = poleAngularDistance / Math.PI;
|
|
397
|
+
observations[2] = cartVelocity;
|
|
398
|
+
observations[3] = poleAngularVelocity;
|
|
399
|
+
|
|
400
|
+
// Apply the actions
|
|
401
|
+
const actions = await this.run();
|
|
402
|
+
if (actions !== null) {
|
|
403
|
+
cartMotor.targetVelocity = actions[0];
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
```
|