streaming-deep-rl 0.2.1__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {streaming_deep_rl-0.2.1 → streaming_deep_rl-0.2.4}/PKG-INFO +14 -1
- {streaming_deep_rl-0.2.1 → streaming_deep_rl-0.2.4}/README.md +12 -0
- {streaming_deep_rl-0.2.1 → streaming_deep_rl-0.2.4}/pyproject.toml +2 -1
- {streaming_deep_rl-0.2.1 → streaming_deep_rl-0.2.4}/streaming_deep_rl/streaming_deep_rl.py +84 -8
- {streaming_deep_rl-0.2.1 → streaming_deep_rl-0.2.4}/.gitignore +0 -0
- {streaming_deep_rl-0.2.1 → streaming_deep_rl-0.2.4}/LICENSE +0 -0
- {streaming_deep_rl-0.2.1 → streaming_deep_rl-0.2.4}/streaming_deep_rl/__init__.py +0 -0
- {streaming_deep_rl-0.2.1 → streaming_deep_rl-0.2.4}/streaming_deep_rl/buffer_dict.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: streaming-deep-rl
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.4
|
|
4
4
|
Summary: Streaming Deep Reinforcement Learning
|
|
5
5
|
Project-URL: Homepage, https://pypi.org/project/streaming-deep-rl/
|
|
6
6
|
Project-URL: Repository, https://codeberg.org/lucidrains/streaming-deep-rl
|
|
@@ -36,6 +36,7 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
|
36
36
|
Requires-Python: >=3.9
|
|
37
37
|
Requires-Dist: discrete-continuous-embed-readout>=0.2.0
|
|
38
38
|
Requires-Dist: einops>=0.8.1
|
|
39
|
+
Requires-Dist: einx>=0.3.0
|
|
39
40
|
Requires-Dist: ema-pytorch
|
|
40
41
|
Requires-Dist: hl-gauss-pytorch
|
|
41
42
|
Requires-Dist: torch-einops-utils
|
|
@@ -228,3 +229,15 @@ $ uv run train_lunar.py
|
|
|
228
229
|
url = {https://arxiv.org/abs/2602.09396},
|
|
229
230
|
}
|
|
230
231
|
```
|
|
232
|
+
|
|
233
|
+
```bibtex
|
|
234
|
+
@misc{maes2026leworldmodelstableendtoendjointembedding,
|
|
235
|
+
title = {LeWorldModel: Stable End-to-End Joint-Embedding Predictive Architecture from Pixels},
|
|
236
|
+
author = {Lucas Maes and Quentin Le Lidec and Damien Scieur and Yann LeCun and Randall Balestriero},
|
|
237
|
+
year = {2026},
|
|
238
|
+
eprint = {2603.19312},
|
|
239
|
+
archivePrefix = {arXiv},
|
|
240
|
+
primaryClass = {cs.LG},
|
|
241
|
+
url = {https://arxiv.org/abs/2603.19312},
|
|
242
|
+
}
|
|
243
|
+
```
|
|
@@ -179,3 +179,15 @@ $ uv run train_lunar.py
|
|
|
179
179
|
url = {https://arxiv.org/abs/2602.09396},
|
|
180
180
|
}
|
|
181
181
|
```
|
|
182
|
+
|
|
183
|
+
```bibtex
|
|
184
|
+
@misc{maes2026leworldmodelstableendtoendjointembedding,
|
|
185
|
+
title = {LeWorldModel: Stable End-to-End Joint-Embedding Predictive Architecture from Pixels},
|
|
186
|
+
author = {Lucas Maes and Quentin Le Lidec and Damien Scieur and Yann LeCun and Randall Balestriero},
|
|
187
|
+
year = {2026},
|
|
188
|
+
eprint = {2603.19312},
|
|
189
|
+
archivePrefix = {arXiv},
|
|
190
|
+
primaryClass = {cs.LG},
|
|
191
|
+
url = {https://arxiv.org/abs/2603.19312},
|
|
192
|
+
}
|
|
193
|
+
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "streaming-deep-rl"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.4"
|
|
4
4
|
description = "Streaming Deep Reinforcement Learning"
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Phil Wang", email = "lucidrains@gmail.com" }
|
|
@@ -25,6 +25,7 @@ classifiers=[
|
|
|
25
25
|
|
|
26
26
|
dependencies = [
|
|
27
27
|
"discrete-continuous-embed-readout>=0.2.0",
|
|
28
|
+
"einx>=0.3.0",
|
|
28
29
|
"einops>=0.8.1",
|
|
29
30
|
"ema-pytorch",
|
|
30
31
|
"hl-gauss-pytorch",
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
import math
|
|
3
3
|
from collections import deque
|
|
4
|
+
from contextlib import nullcontext
|
|
4
5
|
from typing import Callable, NamedTuple
|
|
5
6
|
|
|
6
7
|
import torch
|
|
@@ -8,11 +9,12 @@ import torch.nn.functional as F
|
|
|
8
9
|
from torch import nn, tensor, is_tensor, cat, stack
|
|
9
10
|
from torch.nn import Module, Linear, Sequential
|
|
10
11
|
|
|
11
|
-
|
|
12
|
+
import einx
|
|
13
|
+
from einops import rearrange, reduce
|
|
12
14
|
|
|
13
15
|
from discrete_continuous_embed_readout import Readout, Embed
|
|
14
16
|
|
|
15
|
-
from torch_einops_utils import tree_map_tensor, pack_with_inverse
|
|
17
|
+
from torch_einops_utils import tree_map_tensor, pack_with_inverse, masked_mean
|
|
16
18
|
|
|
17
19
|
from ema_pytorch import EMA
|
|
18
20
|
|
|
@@ -25,12 +27,21 @@ from streaming_deep_rl.buffer_dict import BufferDict
|
|
|
25
27
|
def exists(v):
|
|
26
28
|
return v is not None
|
|
27
29
|
|
|
30
|
+
def identity(t):
|
|
31
|
+
return t
|
|
32
|
+
|
|
28
33
|
def divisible_by(num, den):
|
|
29
34
|
return (num % den) == 0
|
|
30
35
|
|
|
31
36
|
def default(v, d):
|
|
32
37
|
return v if exists(v) else d
|
|
33
38
|
|
|
39
|
+
def no_grad_detach(fn):
|
|
40
|
+
def inner(t):
|
|
41
|
+
with torch.no_grad():
|
|
42
|
+
return fn(t).detach()
|
|
43
|
+
return inner
|
|
44
|
+
|
|
34
45
|
def cast_tensor(t):
|
|
35
46
|
return tensor(t) if not is_tensor(t) else t
|
|
36
47
|
|
|
@@ -104,7 +115,9 @@ class SelfPredictRepr(Module):
|
|
|
104
115
|
dim_embed,
|
|
105
116
|
num_discrete_actions = 0,
|
|
106
117
|
num_continuous_actions = 0,
|
|
107
|
-
dim_predict = None
|
|
118
|
+
dim_predict = None,
|
|
119
|
+
target_embed_from_ema = True, # whether the target embed is from an EMA, or from the model itself (traditional vs sigreg from lejepa)
|
|
120
|
+
sigreg_weight = 0.
|
|
108
121
|
):
|
|
109
122
|
super().__init__()
|
|
110
123
|
assert num_discrete_actions > 0 or num_continuous_actions > 0
|
|
@@ -113,6 +126,8 @@ class SelfPredictRepr(Module):
|
|
|
113
126
|
|
|
114
127
|
self.to_action_embed = Embed(dim_embed, num_discrete = num_discrete_actions, num_continuous = num_continuous_actions)
|
|
115
128
|
|
|
129
|
+
self.target_embed_from_ema = target_embed_from_ema
|
|
130
|
+
|
|
116
131
|
self.to_projection = nn.Sequential(
|
|
117
132
|
nn.RMSNorm(dim_embed),
|
|
118
133
|
nn.Linear(dim_embed, dim_predict, bias = False)
|
|
@@ -122,6 +137,9 @@ class SelfPredictRepr(Module):
|
|
|
122
137
|
nn.Linear(dim_predict + dim_embed, dim_predict, bias = False)
|
|
123
138
|
)
|
|
124
139
|
|
|
140
|
+
self.apply_sigreg = sigreg_weight > 0.
|
|
141
|
+
self.sigreg_weight = sigreg_weight
|
|
142
|
+
|
|
125
143
|
def forward(
|
|
126
144
|
self,
|
|
127
145
|
state_embed,
|
|
@@ -130,8 +148,11 @@ class SelfPredictRepr(Module):
|
|
|
130
148
|
):
|
|
131
149
|
projected = self.to_projection(state_embed)
|
|
132
150
|
|
|
133
|
-
|
|
134
|
-
|
|
151
|
+
# maybe no
|
|
152
|
+
|
|
153
|
+
decorator = no_grad_detach if self.target_embed_from_ema else identity
|
|
154
|
+
|
|
155
|
+
next_projected = decorator(self.to_projection)(next_state_embed)
|
|
135
156
|
|
|
136
157
|
action_embed = self.to_action_embed(actions)
|
|
137
158
|
|
|
@@ -139,7 +160,54 @@ class SelfPredictRepr(Module):
|
|
|
139
160
|
|
|
140
161
|
# cosine sim loss
|
|
141
162
|
|
|
142
|
-
|
|
163
|
+
loss = F.mse_loss(l2norm(predicted), l2norm(next_projected))
|
|
164
|
+
|
|
165
|
+
if not self.apply_sigreg:
|
|
166
|
+
return loss
|
|
167
|
+
|
|
168
|
+
return loss + self.sigreg_weight * sigreg_loss(projected)
|
|
169
|
+
|
|
170
|
+
# sigreg from lejepa
|
|
171
|
+
|
|
172
|
+
def sigreg_loss(
|
|
173
|
+
x,
|
|
174
|
+
num_slices = 1024,
|
|
175
|
+
domain = (-5, 5),
|
|
176
|
+
num_knots = 17,
|
|
177
|
+
mask = None
|
|
178
|
+
):
|
|
179
|
+
# Randall Balestriero - https://arxiv.org/abs/2511.08544
|
|
180
|
+
|
|
181
|
+
dim, device = x.shape[-1], x.device
|
|
182
|
+
|
|
183
|
+
# slice sampling
|
|
184
|
+
|
|
185
|
+
rand_projs = torch.randn((num_slices, dim), device = device)
|
|
186
|
+
rand_projs = l2norm(rand_projs)
|
|
187
|
+
|
|
188
|
+
# integration points
|
|
189
|
+
|
|
190
|
+
t = torch.linspace(*domain, num_knots, device = device)
|
|
191
|
+
|
|
192
|
+
# theoretical CF for N(0, 1) and Gauss. window
|
|
193
|
+
|
|
194
|
+
exp_f = (-0.5 * t.square()).exp()
|
|
195
|
+
|
|
196
|
+
# empirical CF
|
|
197
|
+
|
|
198
|
+
x_t = einx.dot('... d, m d -> (...) m', x, rand_projs)
|
|
199
|
+
|
|
200
|
+
x_t = einx.multiply('n m, t -> n m t', x_t, t)
|
|
201
|
+
ecf = (1j * x_t).exp()
|
|
202
|
+
|
|
203
|
+
mask_1d = rearrange(mask, '... -> (...)') if exists(mask) else None
|
|
204
|
+
ecf = masked_mean(ecf, mask_1d, dim = 0)
|
|
205
|
+
|
|
206
|
+
# weighted L2 distance
|
|
207
|
+
|
|
208
|
+
err = ecf.sub(exp_f).abs().square().mul(exp_f)
|
|
209
|
+
|
|
210
|
+
return torch.trapz(err, t, dim = -1).mean()
|
|
143
211
|
|
|
144
212
|
# online normalization from Welford in 1962
|
|
145
213
|
|
|
@@ -316,6 +384,8 @@ class StreamingACLambda(Module):
|
|
|
316
384
|
critic_ema_beta = 0.7,
|
|
317
385
|
actor_self_predict_repr = False,
|
|
318
386
|
critic_self_predict_repr = False,
|
|
387
|
+
spr_target_embed_from_ema = True,
|
|
388
|
+
spr_sigreg_weight = 0.,
|
|
319
389
|
entropy_weight = 0.01,
|
|
320
390
|
init_sparsity = 0.9,
|
|
321
391
|
dim_critic = 128,
|
|
@@ -390,22 +460,28 @@ class StreamingACLambda(Module):
|
|
|
390
460
|
|
|
391
461
|
if actor_self_predict_repr:
|
|
392
462
|
|
|
393
|
-
|
|
463
|
+
if spr_target_embed_from_ema:
|
|
464
|
+
actor_use_ema |= True
|
|
394
465
|
|
|
395
466
|
self.actor_spr = SelfPredictRepr(
|
|
396
467
|
dim_actor,
|
|
397
468
|
num_discrete_actions = num_discrete_actions,
|
|
398
469
|
num_continuous_actions = num_continuous_actions,
|
|
470
|
+
target_embed_from_ema = spr_target_embed_from_ema,
|
|
471
|
+
sigreg_weight = spr_sigreg_weight
|
|
399
472
|
)
|
|
400
473
|
|
|
401
474
|
if critic_self_predict_repr:
|
|
402
475
|
|
|
403
|
-
|
|
476
|
+
if spr_target_embed_from_ema:
|
|
477
|
+
use_critic_ema |= True
|
|
404
478
|
|
|
405
479
|
self.critic_spr = SelfPredictRepr(
|
|
406
480
|
dim_critic,
|
|
407
481
|
num_discrete_actions = num_discrete_actions,
|
|
408
482
|
num_continuous_actions = num_continuous_actions,
|
|
483
|
+
target_embed_from_ema = spr_target_embed_from_ema,
|
|
484
|
+
sigreg_weight = spr_sigreg_weight
|
|
409
485
|
)
|
|
410
486
|
|
|
411
487
|
# actor
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|