streaming-deep-rl 0.2.1__tar.gz → 0.2.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: streaming-deep-rl
3
- Version: 0.2.1
3
+ Version: 0.2.4
4
4
  Summary: Streaming Deep Reinforcement Learning
5
5
  Project-URL: Homepage, https://pypi.org/project/streaming-deep-rl/
6
6
  Project-URL: Repository, https://codeberg.org/lucidrains/streaming-deep-rl
@@ -36,6 +36,7 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
36
36
  Requires-Python: >=3.9
37
37
  Requires-Dist: discrete-continuous-embed-readout>=0.2.0
38
38
  Requires-Dist: einops>=0.8.1
39
+ Requires-Dist: einx>=0.3.0
39
40
  Requires-Dist: ema-pytorch
40
41
  Requires-Dist: hl-gauss-pytorch
41
42
  Requires-Dist: torch-einops-utils
@@ -228,3 +229,15 @@ $ uv run train_lunar.py
228
229
  url = {https://arxiv.org/abs/2602.09396},
229
230
  }
230
231
  ```
232
+
233
+ ```bibtex
234
+ @misc{maes2026leworldmodelstableendtoendjointembedding,
235
+ title = {LeWorldModel: Stable End-to-End Joint-Embedding Predictive Architecture from Pixels},
236
+ author = {Lucas Maes and Quentin Le Lidec and Damien Scieur and Yann LeCun and Randall Balestriero},
237
+ year = {2026},
238
+ eprint = {2603.19312},
239
+ archivePrefix = {arXiv},
240
+ primaryClass = {cs.LG},
241
+ url = {https://arxiv.org/abs/2603.19312},
242
+ }
243
+ ```
@@ -179,3 +179,15 @@ $ uv run train_lunar.py
179
179
  url = {https://arxiv.org/abs/2602.09396},
180
180
  }
181
181
  ```
182
+
183
+ ```bibtex
184
+ @misc{maes2026leworldmodelstableendtoendjointembedding,
185
+ title = {LeWorldModel: Stable End-to-End Joint-Embedding Predictive Architecture from Pixels},
186
+ author = {Lucas Maes and Quentin Le Lidec and Damien Scieur and Yann LeCun and Randall Balestriero},
187
+ year = {2026},
188
+ eprint = {2603.19312},
189
+ archivePrefix = {arXiv},
190
+ primaryClass = {cs.LG},
191
+ url = {https://arxiv.org/abs/2603.19312},
192
+ }
193
+ ```
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "streaming-deep-rl"
3
- version = "0.2.1"
3
+ version = "0.2.4"
4
4
  description = "Streaming Deep Reinforcement Learning"
5
5
  authors = [
6
6
  { name = "Phil Wang", email = "lucidrains@gmail.com" }
@@ -25,6 +25,7 @@ classifiers=[
25
25
 
26
26
  dependencies = [
27
27
  "discrete-continuous-embed-readout>=0.2.0",
28
+ "einx>=0.3.0",
28
29
  "einops>=0.8.1",
29
30
  "ema-pytorch",
30
31
  "hl-gauss-pytorch",
@@ -1,6 +1,7 @@
1
1
  from __future__ import annotations
2
2
  import math
3
3
  from collections import deque
4
+ from contextlib import nullcontext
4
5
  from typing import Callable, NamedTuple
5
6
 
6
7
  import torch
@@ -8,11 +9,12 @@ import torch.nn.functional as F
8
9
  from torch import nn, tensor, is_tensor, cat, stack
9
10
  from torch.nn import Module, Linear, Sequential
10
11
 
11
- from einops import reduce
12
+ import einx
13
+ from einops import rearrange, reduce
12
14
 
13
15
  from discrete_continuous_embed_readout import Readout, Embed
14
16
 
15
- from torch_einops_utils import tree_map_tensor, pack_with_inverse
17
+ from torch_einops_utils import tree_map_tensor, pack_with_inverse, masked_mean
16
18
 
17
19
  from ema_pytorch import EMA
18
20
 
@@ -25,12 +27,21 @@ from streaming_deep_rl.buffer_dict import BufferDict
25
27
  def exists(v):
26
28
  return v is not None
27
29
 
30
+ def identity(t):
31
+ return t
32
+
28
33
  def divisible_by(num, den):
29
34
  return (num % den) == 0
30
35
 
31
36
  def default(v, d):
32
37
  return v if exists(v) else d
33
38
 
39
+ def no_grad_detach(fn):
40
+ def inner(t):
41
+ with torch.no_grad():
42
+ return fn(t).detach()
43
+ return inner
44
+
34
45
  def cast_tensor(t):
35
46
  return tensor(t) if not is_tensor(t) else t
36
47
 
@@ -104,7 +115,9 @@ class SelfPredictRepr(Module):
104
115
  dim_embed,
105
116
  num_discrete_actions = 0,
106
117
  num_continuous_actions = 0,
107
- dim_predict = None
118
+ dim_predict = None,
119
+ target_embed_from_ema = True, # whether the target embed is from an EMA, or from the model itself (traditional vs sigreg from lejepa)
120
+ sigreg_weight = 0.
108
121
  ):
109
122
  super().__init__()
110
123
  assert num_discrete_actions > 0 or num_continuous_actions > 0
@@ -113,6 +126,8 @@ class SelfPredictRepr(Module):
113
126
 
114
127
  self.to_action_embed = Embed(dim_embed, num_discrete = num_discrete_actions, num_continuous = num_continuous_actions)
115
128
 
129
+ self.target_embed_from_ema = target_embed_from_ema
130
+
116
131
  self.to_projection = nn.Sequential(
117
132
  nn.RMSNorm(dim_embed),
118
133
  nn.Linear(dim_embed, dim_predict, bias = False)
@@ -122,6 +137,9 @@ class SelfPredictRepr(Module):
122
137
  nn.Linear(dim_predict + dim_embed, dim_predict, bias = False)
123
138
  )
124
139
 
140
+ self.apply_sigreg = sigreg_weight > 0.
141
+ self.sigreg_weight = sigreg_weight
142
+
125
143
  def forward(
126
144
  self,
127
145
  state_embed,
@@ -130,8 +148,11 @@ class SelfPredictRepr(Module):
130
148
  ):
131
149
  projected = self.to_projection(state_embed)
132
150
 
133
- with torch.no_grad():
134
- next_projected = self.to_projection(next_state_embed.detach())
151
+ # maybe no
152
+
153
+ decorator = no_grad_detach if self.target_embed_from_ema else identity
154
+
155
+ next_projected = decorator(self.to_projection)(next_state_embed)
135
156
 
136
157
  action_embed = self.to_action_embed(actions)
137
158
 
@@ -139,7 +160,54 @@ class SelfPredictRepr(Module):
139
160
 
140
161
  # cosine sim loss
141
162
 
142
- return F.mse_loss(l2norm(predicted), l2norm(next_projected))
163
+ loss = F.mse_loss(l2norm(predicted), l2norm(next_projected))
164
+
165
+ if not self.apply_sigreg:
166
+ return loss
167
+
168
+ return loss + self.sigreg_weight * sigreg_loss(projected)
169
+
170
+ # sigreg from lejepa
171
+
172
+ def sigreg_loss(
173
+ x,
174
+ num_slices = 1024,
175
+ domain = (-5, 5),
176
+ num_knots = 17,
177
+ mask = None
178
+ ):
179
+ # Randall Balestriero - https://arxiv.org/abs/2511.08544
180
+
181
+ dim, device = x.shape[-1], x.device
182
+
183
+ # slice sampling
184
+
185
+ rand_projs = torch.randn((num_slices, dim), device = device)
186
+ rand_projs = l2norm(rand_projs)
187
+
188
+ # integration points
189
+
190
+ t = torch.linspace(*domain, num_knots, device = device)
191
+
192
+ # theoretical CF for N(0, 1) and Gauss. window
193
+
194
+ exp_f = (-0.5 * t.square()).exp()
195
+
196
+ # empirical CF
197
+
198
+ x_t = einx.dot('... d, m d -> (...) m', x, rand_projs)
199
+
200
+ x_t = einx.multiply('n m, t -> n m t', x_t, t)
201
+ ecf = (1j * x_t).exp()
202
+
203
+ mask_1d = rearrange(mask, '... -> (...)') if exists(mask) else None
204
+ ecf = masked_mean(ecf, mask_1d, dim = 0)
205
+
206
+ # weighted L2 distance
207
+
208
+ err = ecf.sub(exp_f).abs().square().mul(exp_f)
209
+
210
+ return torch.trapz(err, t, dim = -1).mean()
143
211
 
144
212
  # online normalization from Welford in 1962
145
213
 
@@ -316,6 +384,8 @@ class StreamingACLambda(Module):
316
384
  critic_ema_beta = 0.7,
317
385
  actor_self_predict_repr = False,
318
386
  critic_self_predict_repr = False,
387
+ spr_target_embed_from_ema = True,
388
+ spr_sigreg_weight = 0.,
319
389
  entropy_weight = 0.01,
320
390
  init_sparsity = 0.9,
321
391
  dim_critic = 128,
@@ -390,22 +460,28 @@ class StreamingACLambda(Module):
390
460
 
391
461
  if actor_self_predict_repr:
392
462
 
393
- actor_use_ema |= True
463
+ if spr_target_embed_from_ema:
464
+ actor_use_ema |= True
394
465
 
395
466
  self.actor_spr = SelfPredictRepr(
396
467
  dim_actor,
397
468
  num_discrete_actions = num_discrete_actions,
398
469
  num_continuous_actions = num_continuous_actions,
470
+ target_embed_from_ema = spr_target_embed_from_ema,
471
+ sigreg_weight = spr_sigreg_weight
399
472
  )
400
473
 
401
474
  if critic_self_predict_repr:
402
475
 
403
- use_critic_ema |= True
476
+ if spr_target_embed_from_ema:
477
+ use_critic_ema |= True
404
478
 
405
479
  self.critic_spr = SelfPredictRepr(
406
480
  dim_critic,
407
481
  num_discrete_actions = num_discrete_actions,
408
482
  num_continuous_actions = num_continuous_actions,
483
+ target_embed_from_ema = spr_target_embed_from_ema,
484
+ sigreg_weight = spr_sigreg_weight
409
485
  )
410
486
 
411
487
  # actor