metacontroller-pytorch 0.0.1__py3-none-any.whl

metacontroller/__init__.py

@@ -0,0 +1 @@
+from metacontroller.metacontroller import MetaController

metacontroller/metacontroller.py

@@ -0,0 +1,257 @@
+from __future__ import annotations
+from functools import partial
+
+import torch
+from torch import nn, cat, stack, tensor
+from torch.nn import Module, GRU, Linear, Identity
+import torch.nn.functional as F
+
+# einops
+
+import einx
+from einops import einsum, rearrange, repeat, reduce
+from einops.layers.torch import Rearrange
+
+# external modules
+
+from x_transformers import Decoder
+from x_mlps_pytorch import Feedforwards
+from x_evolution import EvoStrategy
+
+from discrete_continuous_embed_readout import Embed, Readout
+
+from assoc_scan import AssocScan
+
+# constants
+
+LinearNoBias = partial(Linear, bias = False)
+
+GRU = partial(GRU, batch_first = True)
+
+# helper functions
+
+def exists(v):
+    return v is not None
+
+def identity(t):
+    return t
+
+def default(*args):
+    for arg in args:
+        if exists(arg):
+            return arg
+    return None
+
+# meta controller
+
+class MetaController(Module):
+    def __init__(
+        self,
+        dim_latent,
+        *,
+        decoder_expansion_factor = 2.,
+        decoder_depth = 1,
+        hypernetwork_low_rank = 16,
+        assoc_scan_kwargs: dict = dict()
+    ):
+        super().__init__()
+
+        # there are two phases, the first (discovery ssl phase) uses acausal with some ssm i don't really believe in - let's just use a bidirectional GRU as placeholders
+
+        self.bidirectional_temporal_compressor = GRU(dim_latent, dim_latent, bidirectional = True) # revisit naming
+
+        self.emitter = GRU(dim_latent * 2, dim_latent * 2)
+        self.emitter_to_action_mean_log_var = LinearNoBias(dim_latent * 2, dim_latent * 2)
+
+        # internal rl phase substitutes the acausal + emitter with a causal ssm
+
+        self.action_proposer = GRU(dim_latent, dim_latent)
+        self.action_proposer_mean_log_var = LinearNoBias(dim_latent, dim_latent * 2)
+
+        # switching unit
+
+        self.switching_unit = GRU(dim_latent, dim_latent)
+        self.to_switching_unit_beta = nn.Linear(dim_latent, 1, bias = False)
+
+        self.switch_gating = AssocScan(**assoc_scan_kwargs)
+
+        # decoder
+
+        assert hypernetwork_low_rank < dim_latent
+
+        dim_decoder_hidden = int(dim_latent * decoder_expansion_factor)
+
+        self.decoder = Feedforwards(
+            dim_in = dim_latent,
+            dim = dim_decoder_hidden,
+            depth = decoder_depth,
+            dim_out = 2 * hypernetwork_low_rank * dim_latent
+        )
+
+        self.to_hyper_network_weights = Rearrange('... (two d r) -> two ... d r', two = 2, r = hypernetwork_low_rank)
+
+        self.register_buffer('zero', tensor(0.), persistent = False)
+
+    def discovery_parameters(self):
+        return [
+            *self.bidirectional_temporal_compressor.parameters(),
+            *self.emitter.parameters(),
+            *self.emitter_to_action_mean_log_var.parameters()
+            *self.decoder.parameters(),
+            *self.switch_gating
+        ]
+
+    def internal_rl_parameters(self):
+        return [
+            *self.action_proposer.parameters(),
+            *self.action_proposer_mean_log_var.parameters(),
+            *self.decoder.parameters(),
+            *self.switch_gating
+        ]
+
+    def forward(
+        self,
+        residual_stream,
+        discovery_phase = False
+    ):
+
+        if discovery_phase:
+            temporal_compressed, _ = self.bidirectional_temporal_compressor(residual_stream)
+            temporal_compressed = reduce(temporal_compressed, '... (two d) -> ... d', 'mean', two = 2)
+
+            proposed_action_hidden, _ = self.emitter(cat((temporal_compressed, residual_stream), dim = -1))
+            proposed_action = self.emitter_to_action_mean_log_var(proposed_action_hidden)
+
+        else: # else internal rl phase
+            proposed_action_hidden, _ = self.action_proposer(residual_stream)
+            proposed_action = self.action_proposer_mean_log_var(proposed_action_hidden)
+
+        # sample from the gaussian as the action from the meta controller
+
+        mean, log_var = proposed_action.chunk(2, dim = -1)
+
+        std = (0.5 * log_var).exp()
+        sampled_action_intents = mean + torch.randn_like(mean) * std
+
+        # need to encourage normal distribution
+
+        vae_kl_loss = self.zero
+
+        if discovery_phase:
+            vae_kl_loss = (0.5 * (
+                log_var.exp()
+                + mean.square()
+                - log_var
+                - 1.
+            )).sum(dim = -1).mean()
+
+        # switching unit timer
+
+        batch, _, dim = sampled_action_intents.shape
+
+        switching_unit_gru_out, switching_unit_gru_hidden = self.switching_unit(residual_stream)
+
+        switch_beta = self.to_switching_unit_beta(switching_unit_gru_out).sigmoid()
+
+        action_intent_for_gating = rearrange(sampled_action_intents, 'b n d -> (b d) n')
+        switch_beta = repeat(switch_beta, 'b n 1 -> (b d) n', d = dim)
+
+        forget = 1. - switch_beta
+        gated_action_intent = self.switch_gating(action_intent_for_gating * forget, switch_beta)
+
+        gated_action_intent = rearrange(gated_action_intent, '(b d) n -> b n d', b = batch)
+
+        # decoder
+
+        decoder_out = self.decoder(gated_action_intent)
+
+        w1, w2 = self.to_hyper_network_weights(decoder_out)
+        hypernetwork_weight = einsum(w1, w2, '... i r, ... j r -> ... i j')
+
+        # generating the residual stream controlling signal
+
+        control_signal = einsum(gated_action_intent, hypernetwork_weight, '... d1, ... d1 d2 -> ... d1')
+
+        modified_residual_stream = residual_stream + control_signal
+
+        return modified_residual_stream, vae_kl_loss
+
+# main transformer, which is subsumed into the environment after behavioral cloning
+
+class Transformer(Module):
+    def __init__(
+        self,
+        dim,
+        *,
+        embed: Embed | dict,
+        lower_body: Decoder | dict,
+        upper_body: Decoder | dict,
+        readout: Readout | dict,
+        meta_controller: MetaController | None = None
+    ):
+        super().__init__()
+
+        if isinstance(embed, dict):
+            embed = Embed(dim = dim, **embed)
+
+        if isinstance(lower_body, dict):
+            lower_body = Decoder(dim = dim, **lower_body)
+
+        if isinstance(upper_body, dict):
+            upper_body = Decoder(dim = dim, **upper_body)
+
+        if isinstance(readout, dict):
+            readout = Readout(dim = dim, **readout)
+
+        self.embed = embed
+        self.lower_body = lower_body
+        self.upper_body = upper_body
+        self.readout = readout
+
+        # meta controller
+
+        self.meta_controller = meta_controller
+
+    def evolve(
+        self,
+        environment,
+        **kwargs
+    ):
+        assert exists(self.meta_controller), '`meta_controller` must be defined on init for evolutionary strategies to be straightforwardly applied'
+
+        evo_strat = EvoStrategy(
+            self,
+            environment = environment,
+            params_to_optimize = self.meta_controller.internal_rl_parameters(),
+            **kwargs
+        )
+
+        evo_strat()
+
+    def forward(
+        self,
+        ids,
+        meta_controller: Module | None = None,
+        discovery_phase = False,
+        return_latents = False
+    ):
+        meta_controller = default(meta_controller, self.meta_controller, Identity())
+
+        embed = self.embed(ids)
+
+        residual_stream = self.lower_body(embed)
+
+        # meta controller acts on residual stream here
+
+        modified_residual_stream, vae_aux_loss = meta_controller(residual_stream, discovery_phase = discovery_phase)
+
+        # modified residual stream sent back
+
+        attended = self.upper_body(modified_residual_stream)
+
+        dist_params = self.readout(attended)
+
+        if not return_latents:
+            return dist_params
+
+        return dist_params, latents

metacontroller_pytorch-0.0.1.dist-info/METADATA

@@ -0,0 +1,79 @@
+Metadata-Version: 2.4
+Name: metacontroller-pytorch
+Version: 0.0.1
+Summary: Transformer Metacontroller
+Project-URL: Homepage, https://pypi.org/project/metacontroller/
+Project-URL: Repository, https://github.com/lucidrains/metacontroller
+Author-email: Phil Wang <lucidrains@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2025 Phil Wang
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: artificial intelligence,deep learning,hierarchical reinforcement learning,latent steering
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Requires-Dist: assoc-scan
+Requires-Dist: discrete-continuous-embed-readout>=0.1.11
+Requires-Dist: einops>=0.8.1
+Requires-Dist: einx>=0.3.0
+Requires-Dist: torch>=2.5
+Requires-Dist: x-evolution>=0.1.23
+Requires-Dist: x-mlps-pytorch
+Requires-Dist: x-transformers
+Provides-Extra: examples
+Provides-Extra: test
+Requires-Dist: pytest; extra == 'test'
+Description-Content-Type: text/markdown
+
+<img src="./fig1.png" width="400px"></img>
+
+## metacontroller (wip)
+
+Implementation of the MetaController proposed in [Emergent temporal abstractions in autoregressive models enable hierarchical reinforcement learning](https://arxiv.org/abs/2512.20605)
+
+## Citations
+
+```bibtex
+@misc{kobayashi2025emergenttemporalabstractionsautoregressive,
+    title   = {Emergent temporal abstractions in autoregressive models enable hierarchical reinforcement learning}, 
+    author  = {Seijin Kobayashi and Yanick Schimpf and Maximilian Schlegel and Angelika Steger and Maciej Wolczyk and Johannes von Oswald and Nino Scherrer and Kaitlin Maile and Guillaume Lajoie and Blake A. Richards and Rif A. Saurous and James Manyika and Blaise Agüera y Arcas and Alexander Meulemans and João Sacramento},
+    year={2025},
+    eprint  = {2512.20605},
+    archivePrefix = {arXiv},
+    primaryClass = {cs.LG},
+    url     = {https://arxiv.org/abs/2512.20605}, 
+}
+```
+
+```bibtex
+@article{Wagenmaker2025SteeringYD,
+    title   = {Steering Your Diffusion Policy with Latent Space Reinforcement Learning},
+    author  = {Andrew Wagenmaker and Mitsuhiko Nakamoto and Yunchu Zhang and Seohong Park and Waleed Yagoub and Anusha Nagabandi and Abhishek Gupta and Sergey Levine},
+    journal = {ArXiv},
+    year    = {2025},
+    volume  = {abs/2506.15799},
+    url     = {https://api.semanticscholar.org/CorpusID:279464702}
+}
+```

metacontroller_pytorch-0.0.1.dist-info/RECORD

@@ -0,0 +1,6 @@
+metacontroller/__init__.py,sha256=lj7IOGpN--qMxJWbB-4SGqoPXG7Hd4mgtToTRSyTZ58,57
+metacontroller/metacontroller.py,sha256=Pyv6iNGj8qyNKTF8AVHo9MK-Jg8g7A0xo5jZTeSU4Ys,7717
+metacontroller_pytorch-0.0.1.dist-info/METADATA,sha256=PuTRLQAP7vDagLO4f_bSQJJDS1tbhltwcczoQg_QZzk,3706
+metacontroller_pytorch-0.0.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+metacontroller_pytorch-0.0.1.dist-info/licenses/LICENSE,sha256=1yCiA9b5nhslTavxPjsQAO-wpOnwJR9-l8LTVi7GJuk,1066
+metacontroller_pytorch-0.0.1.dist-info/RECORD,,

metacontroller_pytorch-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

metacontroller_pytorch-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Phil Wang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.