model-unfolder 0.2.4__tar.gz → 0.2.6__tar.gz

{model_unfolder-0.2.4 → model_unfolder-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: model-unfolder
-Version: 0.2.4
+Version: 0.2.6
 Summary: Unfold any HuggingFace transformer into an interactive architecture diagram, inline in Jupyter.
 Author: model-unfolder contributors
 License: Apache-2.0
@@ -108,11 +108,18 @@ Param estimates are close to published numbers — DeepSeek-V3 reports `~675B (~
 
 | Family | Models |
 |---|---|
-| DeepSeek | DeepSeek-V2, DeepSeek-V3, Kimi K2 |
-| Llama | Llama 3 / 3.1 / 3.2 / 3.3 |
+| DeepSeek | DeepSeek-V2, DeepSeek-V3 (+ MTP head), Kimi K2 |
+| Llama | Llama 3 / 3.1 / 3.2 / 3.3, OLMo-2, Llama 4 Scout / Maverick (MoE + iRoPE NoPE layers) |
 | Mistral | Mistral 7B, Mixtral 8x7B / 8x22B, Mistral Medium 3.5 |
-| Qwen | Qwen2 / 2.5, Qwen2-MoE, Qwen3, Qwen3-MoE, Qwen3.5 / 3.6 (hybrid linear+full attn) |
-| Gemma | Gemma 3, Gemma 4 (31B, E2B, E4B) |
+| Qwen | Qwen2 / 2.5, Qwen2-MoE, Qwen3, Qwen3-MoE, Qwen3.5 / 3.6 (+ MTP) |
+| Gemma | Gemma 2 9B / 27B (interleaved local+global), Gemma 3 / 3n (+ PLE), Gemma 4 31B / E2B / E4B (+ PLE), RecurrentGemma 2B / 9B (LRU + local attention) |
+| Cohere | Command R, Command R+, Command R7B (QK-Norm attention) |
+| Jamba | Jamba (SSM + attention hybrid, MoE) |
+| Zamba | Zamba 7B, Zamba2 2.7B / 7B (Mamba SSM + weight-shared attention) |
+| Mamba | Mamba 130M–2.8B, Mamba-2 (pure SSM, no attention) |
+| Falcon | Falcon 7B / 40B (parallel attn+FFN), Falcon-H1 (Mamba-2 SSM) |
+| MiniMax | MiniMax-Text-01 (lightning + softmax hybrid, MoE) |
+| RWKV | RWKV-4 / 5 / 6 (pure recurrent, no attention) |
 
 ### Diffusors
 

{model_unfolder-0.2.4 → model_unfolder-0.2.6}/README.md

@@ -84,11 +84,18 @@ Param estimates are close to published numbers — DeepSeek-V3 reports `~675B (~
 
 | Family | Models |
 |---|---|
-| DeepSeek | DeepSeek-V2, DeepSeek-V3, Kimi K2 |
-| Llama | Llama 3 / 3.1 / 3.2 / 3.3 |
+| DeepSeek | DeepSeek-V2, DeepSeek-V3 (+ MTP head), Kimi K2 |
+| Llama | Llama 3 / 3.1 / 3.2 / 3.3, OLMo-2, Llama 4 Scout / Maverick (MoE + iRoPE NoPE layers) |
 | Mistral | Mistral 7B, Mixtral 8x7B / 8x22B, Mistral Medium 3.5 |
-| Qwen | Qwen2 / 2.5, Qwen2-MoE, Qwen3, Qwen3-MoE, Qwen3.5 / 3.6 (hybrid linear+full attn) |
-| Gemma | Gemma 3, Gemma 4 (31B, E2B, E4B) |
+| Qwen | Qwen2 / 2.5, Qwen2-MoE, Qwen3, Qwen3-MoE, Qwen3.5 / 3.6 (+ MTP) |
+| Gemma | Gemma 2 9B / 27B (interleaved local+global), Gemma 3 / 3n (+ PLE), Gemma 4 31B / E2B / E4B (+ PLE), RecurrentGemma 2B / 9B (LRU + local attention) |
+| Cohere | Command R, Command R+, Command R7B (QK-Norm attention) |
+| Jamba | Jamba (SSM + attention hybrid, MoE) |
+| Zamba | Zamba 7B, Zamba2 2.7B / 7B (Mamba SSM + weight-shared attention) |
+| Mamba | Mamba 130M–2.8B, Mamba-2 (pure SSM, no attention) |
+| Falcon | Falcon 7B / 40B (parallel attn+FFN), Falcon-H1 (Mamba-2 SSM) |
+| MiniMax | MiniMax-Text-01 (lightning + softmax hybrid, MoE) |
+| RWKV | RWKV-4 / 5 / 6 (pure recurrent, no attention) |
 
 ### Diffusors
 

{model_unfolder-0.2.4 → model_unfolder-0.2.6}/model_unfolder/adapters/transformer/assembly.py

@@ -5,7 +5,7 @@ from collections.abc import Iterable, Mapping
 from typing import Any
 
 from ...ir import AttentionSpec, FFNSpec, LayerSpec
-from .blocks import decoder_layer_blocks, decoder_only_render_spec
+from .blocks import decoder_layer_blocks, decoder_only_render_spec, parallel_decoder_layer_blocks
 
 
 def decoder_layer(
@@ -15,15 +15,43 @@ def decoder_layer(
     hidden_size: int,
     *,
     extra_blocks: Iterable[dict] | None = None,
+    norm_kind: str = "rmsnorm",
+    norm_placement: str = "pre",
 ) -> LayerSpec:
     """Build a decoder layer from parsed specs plus optional reusable parts."""
-    blocks = decoder_layer_blocks(attention, ffn, hidden_size)
+    blocks = decoder_layer_blocks(attention, ffn, hidden_size, norm_kind=norm_kind)
     if extra_blocks:
         blocks.extend(extra_blocks)
     return LayerSpec(
         index=index,
         attention=attention,
         ffn=ffn,
+        norm_kind=norm_kind,
+        norm_placement=norm_placement,
+        blocks=blocks,
+    )
+
+
+def parallel_decoder_layer(
+    index: int,
+    attention: AttentionSpec,
+    ffn: FFNSpec,
+    hidden_size: int,
+    *,
+    norm_kind: str = "rmsnorm",
+) -> LayerSpec:
+    """Build a parallel-residual decoder layer (GPT-NeoX / GPT-J).
+
+    Attention and FFN share a single input norm and their outputs are summed
+    into one residual add rather than two sequential adds.
+    """
+    blocks = parallel_decoder_layer_blocks(attention, ffn, hidden_size, norm_kind=norm_kind)
+    return LayerSpec(
+        index=index,
+        attention=attention,
+        ffn=ffn,
+        norm_kind=norm_kind,
+        norm_placement="pre",
         blocks=blocks,
     )
 

model_unfolder-0.2.6/model_unfolder/adapters/transformer/blocks/__init__.py

@@ -0,0 +1,56 @@
+"""Reusable transformer block descriptions for renderers.
+
+Adapters attach these block parts to the IR. Renderers can then draw generic
+decoder-only transformer layouts without rediscovering model-specific names
+or labels with another layer of ``if model_type`` logic.
+
+The package is split by responsibility:
+
+* ``model``: model-level input/output bookend blocks.
+* ``layers``: sequential and parallel decoder-layer topology.
+* ``attention``: reusable attention/SSM/recurrent child block contracts.
+* ``feed_forward``: dense, gated, and MoE FFN child block contracts.
+* ``descriptions``: labels, titles, and short metadata strings.
+
+Each block carries two orthogonal tags:
+
+* ``role`` — semantic ("norm", "attention", "ffn", "residual", "gate") used
+  for tooltips, click handlers, and the inspect cards.
+* ``kind`` — rendering shape ("norm", "linear", "activation", "attention",
+  "ffn", "residual_add", "gate_mul", "embedding", "output", "source") used
+  by the architecture view to pick a glyph and lay out a slot.
+
+Edges between blocks travel on the destination side as plain string fields:
+
+* ``residual_from: "<other_block_id>"`` — the residual_add block consumes the
+  *input* of the named block (the standard pre-attention bypass pattern).
+* ``lane: "left" | "right"`` — the block is rendered off the central chain
+  and connected via ``tap_from`` / ``feeds``. Reusable parts such as
+  per-layer embeddings use this instead of model-specific renderer logic.
+"""
+from __future__ import annotations
+
+from .attention import attention_child_blocks
+from .descriptions import (
+    attention_label,
+    attention_title,
+    describe_attention,
+    describe_ffn,
+)
+from .feed_forward import ffn_child_blocks, ffn_detail_view
+from .layers import decoder_layer_blocks, parallel_decoder_layer_blocks
+from .model import decoder_model_blocks, decoder_only_render_spec
+
+__all__ = [
+    "attention_child_blocks",
+    "attention_label",
+    "attention_title",
+    "decoder_layer_blocks",
+    "decoder_model_blocks",
+    "decoder_only_render_spec",
+    "describe_attention",
+    "describe_ffn",
+    "ffn_child_blocks",
+    "ffn_detail_view",
+    "parallel_decoder_layer_blocks",
+]

model_unfolder-0.2.6/model_unfolder/adapters/transformer/blocks/attention.py

@@ -0,0 +1,298 @@
+"""Reusable attention-family child block declarations."""
+from __future__ import annotations
+
+from ....ir import AttentionSpec
+from ..common import format_dim as _fmt
+
+
+def attention_child_blocks(attention: AttentionSpec, hidden_size: int) -> list[dict]:
+    builders = {
+        "mla": _mla_child_blocks,
+        "ssm": _ssm_child_blocks,
+        "recurrent": _recurrent_child_blocks,
+        "rwkv": _rwkv_child_blocks,
+        "linear": _linear_attention_child_blocks,
+    }
+    builder = builders.get(attention.kind, _sdpa_child_blocks)
+    return builder(attention, hidden_size)
+
+
+def _sdpa_child_blocks(attention: AttentionSpec, hidden_size: int) -> list[dict]:
+    hidden = _fmt(hidden_size)
+    num_heads = attention.num_heads or 0
+    num_kv_heads = attention.num_kv_heads or num_heads
+    head_dim = attention.head_dim or 0
+    q_per_group = num_heads // num_kv_heads if (num_heads and num_kv_heads and num_heads % num_kv_heads == 0) else None
+    q_out = _fmt(num_heads * head_dim) if (num_heads and head_dim) else hidden
+    kv_out = _fmt(num_kv_heads * head_dim) if (num_kv_heads and head_dim) else hidden
+    d_k = _fmt(head_dim) if head_dim else "d_k"
+    attention_title, attention_desc = _sdpa_operation_meta(attention, num_heads, num_kv_heads, d_k, q_per_group)
+    return [
+        {
+            "id": "q_proj",
+            "title": "Query projection",
+            "description": f"Linear; {hidden} -> {q_out}  ({num_heads} heads x {d_k} dims)",
+        },
+        {
+            "id": "k_proj",
+            "title": "Key projection",
+            "description": f"Linear; {hidden} -> {kv_out}  ({num_kv_heads} KV-heads x {d_k} dims)",
+        },
+        {
+            "id": "v_proj",
+            "title": "Value projection",
+            "description": f"Linear; {hidden} -> {kv_out}  ({num_kv_heads} KV-heads x {d_k} dims)",
+        },
+        {
+            "id": "qkv_dot",
+            "title": attention_title,
+            "description": attention_desc,
+        },
+        {
+            "id": "o_proj",
+            "title": "Output projection",
+            "description": f"Linear; {q_out} -> {hidden}  (recombines all {num_heads} heads)",
+        },
+    ]
+
+
+def _sdpa_operation_meta(
+    attention: AttentionSpec,
+    num_heads: int,
+    num_kv_heads: int,
+    d_k: str,
+    q_per_group: int | None,
+) -> tuple[str, str]:
+    if attention.kind == "mqa":
+        return (
+            "Multi-query scaled dot-product attention",
+            (
+                f"scores = softmax(QK^T / sqrt({d_k})); "
+                f"{num_heads} query heads share one K/V head"
+            ),
+        )
+    if attention.kind == "gqa":
+        group = (
+            f"; each KV head serves {q_per_group} query heads"
+            if q_per_group
+            else ""
+        )
+        return (
+            "Grouped scaled dot-product attention",
+            (
+                f"scores = softmax(QK^T / sqrt({d_k})); "
+                f"{num_heads} query heads attend through {num_kv_heads} shared KV heads{group}"
+            ),
+        )
+    return (
+        "Scaled dot-product attention",
+        (
+            f"scores = softmax(QK^T / sqrt({d_k})); "
+            "context = scores * V; "
+            f"output shape [batch, {num_heads}, seq, {d_k}]"
+        ),
+    )
+
+
+def _mla_child_blocks(attention: AttentionSpec, hidden_size: int) -> list[dict]:
+    hidden = _fmt(hidden_size)
+    q_rank = _fmt(attention.q_lora_rank) if attention.q_lora_rank else "direct"
+    kv_rank = _fmt(attention.kv_lora_rank)
+    rope = _fmt(attention.rope_dim)
+    num_heads = attention.num_heads or 0
+    head_dim = attention.head_dim or 0
+    q_out = _fmt(num_heads * head_dim) if (num_heads and head_dim) else hidden
+    return [
+        {
+            "id": "mla_q",
+            "label": "Q projection",
+            "title": "Query projection",
+            "description": (
+                f"Projects hidden states into query heads through LoRA rank {q_rank}"
+                if attention.q_lora_rank
+                else f"Q projection; {hidden} -> {q_out}"
+            ),
+        },
+        {
+            "id": "mla_kv_down",
+            "label": "KV compress",
+            "title": "K/V latent compression",
+            "description": f"Compresses the token state into a shared latent K/V vector; {hidden} -> rank {kv_rank}",
+        },
+        {
+            "id": "mla_kv_up",
+            "label": "KV expand",
+            "title": "K/V head expansion",
+            "description": f"Expands the latent K/V vector into per-head key/value content for {num_heads} query heads",
+        },
+        {
+            "id": "mla_rope",
+            "label": "RoPE key",
+            "title": "Rotary key side-channel",
+            "description": f"Separate positional key slice used with RoPE; dim {rope}",
+        },
+        {
+            "id": "mla_attn",
+            "label": "Latent attention",
+            "title": "Multi-head latent attention",
+            "description": "Attention over decompressed latent K/V plus the RoPE side channel",
+        },
+        {
+            "id": "o_proj",
+            "label": "Linear (out)",
+            "title": "Output projection",
+            "description": f"Linear; {q_out} -> {hidden}",
+        },
+    ]
+
+
+def _ssm_child_blocks(attention: AttentionSpec, hidden_size: int) -> list[dict]:
+    hidden = _fmt(hidden_size)
+    state = _fmt(attention.head_dim)
+    return [
+        {
+            "id": "ssm_in_proj",
+            "label": "Input projection",
+            "title": "SSM input projection",
+            "description": f"Project hidden activations into SSM channels; hidden {hidden}",
+        },
+        {
+            "id": "ssm_conv",
+            "label": "Local conv",
+            "title": "Short convolution",
+            "description": "Depthwise local mixing before the state-space recurrence",
+        },
+        {
+            "id": "ssm_scan",
+            "label": "Selective scan",
+            "title": "Selective state-space scan",
+            "description": f"Token recurrence with state dimension {state}",
+        },
+        {
+            "id": "ssm_gate",
+            "label": "Gate",
+            "title": "SSM gate",
+            "description": "Element-wise gate controlling the recurrent output",
+        },
+        {
+            "id": "ssm_out_proj",
+            "label": "Output projection",
+            "title": "SSM output projection",
+            "description": f"Project SSM channels back to hidden dim {hidden}",
+        },
+    ]
+
+
+def _recurrent_child_blocks(attention: AttentionSpec, hidden_size: int) -> list[dict]:
+    hidden = _fmt(hidden_size)
+    width = _fmt(attention.head_dim)
+    return [
+        {
+            "id": "lru_in_proj",
+            "label": "Input projection",
+            "title": "LRU input projection",
+            "description": f"Linear; hidden {hidden} -> recurrent width {width}",
+        },
+        {
+            "id": "lru_state",
+            "label": "Recurrent state",
+            "title": "Linear recurrent state",
+            "description": f"State update over sequence positions; width {width}",
+        },
+        {
+            "id": "lru_gate",
+            "label": "Gate",
+            "title": "Recurrent gate",
+            "description": "Element-wise gate controlling recurrent features",
+        },
+        {
+            "id": "lru_out_proj",
+            "label": "Output projection",
+            "title": "LRU output projection",
+            "description": f"Linear; recurrent width {width} -> hidden {hidden}",
+        },
+    ]
+
+
+def _rwkv_child_blocks(attention: AttentionSpec, hidden_size: int) -> list[dict]:
+    hidden = _fmt(hidden_size)
+    heads = attention.num_heads or 0
+    return [
+        {
+            "id": "rwkv_receptance",
+            "label": "Receptance",
+            "title": "Receptance gate",
+            "description": f"Token-wise gate over hidden dim {hidden}",
+        },
+        {
+            "id": "rwkv_key",
+            "label": "Key",
+            "title": "RWKV key projection",
+            "description": f"Key-like channel mixing over {heads} recurrent heads",
+        },
+        {
+            "id": "rwkv_value",
+            "label": "Value",
+            "title": "RWKV value projection",
+            "description": "Value channel sent through time-mixing recurrence",
+        },
+        {
+            "id": "rwkv_time_mix",
+            "label": "Time-mix",
+            "title": "Time-decay recurrence",
+            "description": "Linear-time weighted recurrence replacing self-attention",
+        },
+        {
+            "id": "rwkv_out",
+            "label": "Output projection",
+            "title": "RWKV output projection",
+            "description": f"Project mixed channels back to hidden dim {hidden}",
+        },
+    ]
+
+
+def _linear_attention_child_blocks(attention: AttentionSpec, hidden_size: int) -> list[dict]:
+    hidden = _fmt(hidden_size)
+    num_heads = attention.num_heads or 0
+    num_kv_heads = attention.num_kv_heads or num_heads
+    head_dim = attention.head_dim or 0
+    q_out = _fmt(num_heads * head_dim) if (num_heads and head_dim) else hidden
+    kv_out = _fmt(num_kv_heads * head_dim) if (num_kv_heads and head_dim) else hidden
+    return [
+        {
+            "id": "q_proj",
+            "label": "Linear (Q)",
+            "title": "Query projection",
+            "description": f"Linear; {hidden} -> {q_out}",
+        },
+        {
+            "id": "k_proj",
+            "label": "Linear (K)",
+            "title": "Key projection",
+            "description": f"Linear; {hidden} -> {kv_out}",
+        },
+        {
+            "id": "v_proj",
+            "label": "Linear (V)",
+            "title": "Value projection",
+            "description": f"Linear; {hidden} -> {kv_out}",
+        },
+        {
+            "id": "kernel_map",
+            "label": "Kernel map",
+            "title": "Feature map",
+            "description": "Apply kernel feature map so attention can be accumulated linearly",
+        },
+        {
+            "id": "linear_mix",
+            "label": "Linear mix",
+            "title": "Linear attention mix",
+            "description": "Prefix/state accumulation computes attention in linear time",
+        },
+        {
+            "id": "o_proj",
+            "label": "Linear (out)",
+            "title": "Output projection",
+            "description": f"Linear; {q_out} -> {hidden}",
+        },
+    ]

model_unfolder-0.2.6/model_unfolder/adapters/transformer/blocks/descriptions.py

@@ -0,0 +1,132 @@
+"""Labels, titles, and descriptions for transformer block specs."""
+from __future__ import annotations
+
+from ....ir import AttentionSpec, FFNSpec
+from ....labels import activation_label
+from ..common import format_dim as _fmt
+
+
+def attention_label(attention: AttentionSpec) -> list[str]:
+    kind = attention.kind
+    prefix = _attention_mask_prefix(attention)
+    if kind == "mla":
+        return _prefixed_label(prefix, "Multi-Head Latent", "Attention")
+    if kind == "mqa":
+        return _prefixed_label(prefix, "Multi-Query", "Attention")
+    if kind == "gqa":
+        tag = "(QK-Norm)" if attention.qk_norm else "Attention"
+        return _prefixed_label(prefix, "Grouped-Query", tag)
+    if kind == "ssm":
+        shared_tag = "(Shared)" if attention.shared else "Block"
+        return ["Selective SSM", shared_tag]
+    if kind == "recurrent":
+        return ["Linear Recurrent", "Unit (LRU)"]
+    if kind == "rwkv":
+        return ["RWKV", "Token-Mixing"]
+    if kind == "linear":
+        return ["Linear", "Attention"]
+
+    tags = []
+    if attention.qk_norm:
+        tags.append("QK-Norm")
+    if attention.no_rope:
+        tags.append("NoPE")
+    if tags:
+        return ["Multi-Head Attn", f"({', '.join(tags)})"]
+    return ["Multi-Head", "Attention"]
+
+
+def attention_title(attention: AttentionSpec) -> str:
+    if attention.kind == "mqa":
+        base = "Multi-query attention"
+    else:
+        base = {
+            "mla": "Multi-head latent attention",
+            "gqa": "Grouped-query attention",
+            "ssm": "Selective state-space model (Mamba)",
+            "recurrent": "Linear Recurrent Unit (LRU)",
+            "rwkv": "RWKV token-mixing",
+            "linear": "Linear attention",
+        }.get(attention.kind, "Attention")
+    base = _prefixed_title(_attention_mask_title_prefix(attention), base)
+    extras = []
+    if attention.qk_norm:
+        extras.append("QK-Norm")
+    if attention.shared:
+        extras.append("weight-shared")
+    if attention.no_rope:
+        extras.append("NoPE")
+    return f"{base} ({', '.join(extras)})" if extras else base
+
+
+def describe_attention(attention: AttentionSpec) -> str:
+    if attention.kind == "mla":
+        text = (
+            f"Multi-head latent attention; {attention.num_heads} heads; "
+            f"KV LoRA {_fmt(attention.kv_lora_rank)}"
+        )
+        if attention.q_lora_rank:
+            text += f"; Q LoRA {_fmt(attention.q_lora_rank)}"
+        return text
+    if attention.kind == "mqa":
+        return _with_attention_window(attention, f"Multi-query; {attention.num_heads} Q / 1 KV head")
+    if attention.kind == "gqa":
+        return _with_attention_window(attention, (
+            f"Grouped-query; {attention.num_heads} Q / {attention.num_kv_heads} KV heads; "
+            f"head dim {_fmt(attention.head_dim)}"
+        ))
+    if attention.kind == "ssm":
+        shared = "; weight-shared across positions" if attention.shared else ""
+        return f"Selective SSM; state dim {_fmt(attention.head_dim)}{shared}"
+    if attention.kind == "recurrent":
+        return f"Linear Recurrent Unit; LRU width {_fmt(attention.head_dim)}"
+    if attention.kind == "rwkv":
+        return f"RWKV token-mixing; {attention.num_heads} heads"
+    if attention.kind == "linear":
+        return (
+            f"Linear attention; {attention.num_heads} Q / {attention.num_kv_heads} KV; "
+            f"head dim {_fmt(attention.head_dim)}"
+        )
+
+    extras = []
+    if attention.qk_norm:
+        extras.append("QK-Norm")
+    if attention.no_rope:
+        extras.append("NoPE")
+    suffix = f"; {', '.join(extras)}" if extras else ""
+    return _with_attention_window(attention, f"Multi-head; {attention.num_heads} heads; head dim {_fmt(attention.head_dim)}{suffix}")
+
+
+def _attention_mask_prefix(attention: AttentionSpec) -> str:
+    return "SWA" if attention.mask == "sliding" else ""
+
+
+def _attention_mask_title_prefix(attention: AttentionSpec) -> str:
+    return "Sliding-window" if attention.mask == "sliding" else ""
+
+
+def _prefixed_label(prefix: str, first: str, second: str) -> list[str]:
+    return [f"{prefix} · {first}", second] if prefix else [first, second]
+
+
+def _prefixed_title(prefix: str, title: str) -> str:
+    return f"{prefix} {title}" if prefix else title
+
+
+def _with_attention_window(attention: AttentionSpec, text: str) -> str:
+    if attention.mask == "sliding" and attention.window_size:
+        return f"{text}; sliding window {_fmt(attention.window_size)}"
+    return text
+
+
+def describe_ffn(ffn: FFNSpec) -> str:
+    if ffn.kind == "moe":
+        text = f"MoE; {_fmt(ffn.num_experts)} experts; top-{ffn.num_experts_per_tok}"
+        if ffn.num_shared_experts:
+            text += f" + {ffn.num_shared_experts} shared"
+        if ffn.num_experts and ffn.num_experts_per_tok:
+            text += f"; {100 * ffn.num_experts_per_tok / ffn.num_experts:.1f}% active"
+        text += f"; expert hidden {_fmt(ffn.expert_intermediate_size or ffn.intermediate_size)}"
+        return text
+    gated = "gated " if ffn.gated else ""
+    return f"{gated}FFN; {activation_label(ffn.activation)}; hidden {_fmt(ffn.intermediate_size)}"