attnview 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- attnview-0.1.0/LICENSE +21 -0
- attnview-0.1.0/PKG-INFO +95 -0
- attnview-0.1.0/README.md +70 -0
- attnview-0.1.0/pyproject.toml +36 -0
- attnview-0.1.0/setup.cfg +4 -0
- attnview-0.1.0/src/attnview/__init__.py +7 -0
- attnview-0.1.0/src/attnview/api.py +66 -0
- attnview-0.1.0/src/attnview/html.py +75 -0
- attnview-0.1.0/src/attnview/model.py +115 -0
- attnview-0.1.0/src/attnview/terminal.py +65 -0
- attnview-0.1.0/src/attnview.egg-info/PKG-INFO +95 -0
- attnview-0.1.0/src/attnview.egg-info/SOURCES.txt +14 -0
- attnview-0.1.0/src/attnview.egg-info/dependency_links.txt +1 -0
- attnview-0.1.0/src/attnview.egg-info/requires.txt +6 -0
- attnview-0.1.0/src/attnview.egg-info/top_level.txt +1 -0
- attnview-0.1.0/tests/test_attnview.py +69 -0
attnview-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Pedro Gustavo
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
attnview-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: attnview
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Beautiful terminal and HTML views for transformer attention patterns.
|
|
5
|
+
Author-email: Pedro Gustavo <pedrogustavosilva3060@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/pegruk/attnview
|
|
8
|
+
Project-URL: Issues, https://github.com/pegruk/attnview/issues
|
|
9
|
+
Keywords: attention,transformers,interpretability,visualization,terminal
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: rich>=13.7
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
22
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
23
|
+
Requires-Dist: twine>=5.0; extra == "dev"
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# attnview
|
|
27
|
+
|
|
28
|
+
`attnview` is a small, terminal-first viewer for transformer attention patterns.
|
|
29
|
+
It accepts a single attention layer shaped `[n_heads, query_length, key_length]`,
|
|
30
|
+
keeps query and key axes explicit, and renders either a readable terminal heatmap
|
|
31
|
+
or a self-contained interactive HTML report.
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install attnview
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
import attnview
|
|
39
|
+
|
|
40
|
+
# `attention` can be a NumPy array, a PyTorch tensor, or nested Python sequences.
|
|
41
|
+
attnview.show(attention, tokens=["[BOS]", "The", "cat", "sat", "."])
|
|
42
|
+
|
|
43
|
+
# Write a standalone report that can be opened or shared without a server.
|
|
44
|
+
attnview.show(attention, tokens=tokens, output="attention.html")
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Reading the view
|
|
48
|
+
|
|
49
|
+
- **Rows are queries**: the token doing the attending.
|
|
50
|
+
- **Columns are keys**: the token being attended to.
|
|
51
|
+
- The selected head is rendered in the main heatmap; the HTML report also offers
|
|
52
|
+
one-click head selection through miniature maps.
|
|
53
|
+
- Amber marks high attention, teal marks moderate attention, and the darkest cells
|
|
54
|
+
are near zero. This deliberately avoids a rainbow palette that makes magnitude
|
|
55
|
+
difficult to read.
|
|
56
|
+
|
|
57
|
+
## API
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
attnview.show(
|
|
61
|
+
attention,
|
|
62
|
+
tokens=None,
|
|
63
|
+
query_tokens=None,
|
|
64
|
+
key_tokens=None,
|
|
65
|
+
head=0,
|
|
66
|
+
output=None,
|
|
67
|
+
top_k=3,
|
|
68
|
+
title="Attention Explorer",
|
|
69
|
+
)
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Pass `tokens` for self-attention. For cross-attention, pass `query_tokens` and
|
|
73
|
+
`key_tokens` independently. With `output=None` (the default), `show` prints to
|
|
74
|
+
the current terminal. With an `.html` path, it writes a standalone HTML document
|
|
75
|
+
and returns its `Path`.
|
|
76
|
+
|
|
77
|
+
`render_html(...)` returns the standalone HTML string when embedding it elsewhere.
|
|
78
|
+
|
|
79
|
+
## Requirements
|
|
80
|
+
|
|
81
|
+
Python 3.10+ and [Rich](https://github.com/Textualize/rich). NumPy and PyTorch are
|
|
82
|
+
not required: if they are installed, their arrays/tensors are accepted directly.
|
|
83
|
+
|
|
84
|
+
## Development
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
python -m pip install -e '.[dev]'
|
|
88
|
+
python -m pytest
|
|
89
|
+
python -m build
|
|
90
|
+
python -m twine check dist/*
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## License
|
|
94
|
+
|
|
95
|
+
MIT
|
attnview-0.1.0/README.md
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# attnview
|
|
2
|
+
|
|
3
|
+
`attnview` is a small, terminal-first viewer for transformer attention patterns.
|
|
4
|
+
It accepts a single attention layer shaped `[n_heads, query_length, key_length]`,
|
|
5
|
+
keeps query and key axes explicit, and renders either a readable terminal heatmap
|
|
6
|
+
or a self-contained interactive HTML report.
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
pip install attnview
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
```python
|
|
13
|
+
import attnview
|
|
14
|
+
|
|
15
|
+
# `attention` can be a NumPy array, a PyTorch tensor, or nested Python sequences.
|
|
16
|
+
attnview.show(attention, tokens=["[BOS]", "The", "cat", "sat", "."])
|
|
17
|
+
|
|
18
|
+
# Write a standalone report that can be opened or shared without a server.
|
|
19
|
+
attnview.show(attention, tokens=tokens, output="attention.html")
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## Reading the view
|
|
23
|
+
|
|
24
|
+
- **Rows are queries**: the token doing the attending.
|
|
25
|
+
- **Columns are keys**: the token being attended to.
|
|
26
|
+
- The selected head is rendered in the main heatmap; the HTML report also offers
|
|
27
|
+
one-click head selection through miniature maps.
|
|
28
|
+
- Amber marks high attention, teal marks moderate attention, and the darkest cells
|
|
29
|
+
are near zero. This deliberately avoids a rainbow palette that makes magnitude
|
|
30
|
+
difficult to read.
|
|
31
|
+
|
|
32
|
+
## API
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
attnview.show(
|
|
36
|
+
attention,
|
|
37
|
+
tokens=None,
|
|
38
|
+
query_tokens=None,
|
|
39
|
+
key_tokens=None,
|
|
40
|
+
head=0,
|
|
41
|
+
output=None,
|
|
42
|
+
top_k=3,
|
|
43
|
+
title="Attention Explorer",
|
|
44
|
+
)
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Pass `tokens` for self-attention. For cross-attention, pass `query_tokens` and
|
|
48
|
+
`key_tokens` independently. With `output=None` (the default), `show` prints to
|
|
49
|
+
the current terminal. With an `.html` path, it writes a standalone HTML document
|
|
50
|
+
and returns its `Path`.
|
|
51
|
+
|
|
52
|
+
`render_html(...)` returns the standalone HTML string when embedding it elsewhere.
|
|
53
|
+
|
|
54
|
+
## Requirements
|
|
55
|
+
|
|
56
|
+
Python 3.10+ and [Rich](https://github.com/Textualize/rich). NumPy and PyTorch are
|
|
57
|
+
not required: if they are installed, their arrays/tensors are accepted directly.
|
|
58
|
+
|
|
59
|
+
## Development
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
python -m pip install -e '.[dev]'
|
|
63
|
+
python -m pytest
|
|
64
|
+
python -m build
|
|
65
|
+
python -m twine check dist/*
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## License
|
|
69
|
+
|
|
70
|
+
MIT
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "attnview"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Beautiful terminal and HTML views for transformer attention patterns."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
authors = [{name = "Pedro Gustavo", email = "pedrogustavosilva3060@gmail.com"}]
|
|
13
|
+
keywords = ["attention", "transformers", "interpretability", "visualization", "terminal"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Science/Research",
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
20
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
21
|
+
]
|
|
22
|
+
dependencies = ["rich>=13.7"]
|
|
23
|
+
|
|
24
|
+
[project.optional-dependencies]
|
|
25
|
+
dev = ["build>=1.2", "pytest>=8.0", "twine>=5.0"]
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Homepage = "https://github.com/pegruk/attnview"
|
|
29
|
+
Issues = "https://github.com/pegruk/attnview/issues"
|
|
30
|
+
|
|
31
|
+
[tool.setuptools.packages.find]
|
|
32
|
+
where = ["src"]
|
|
33
|
+
|
|
34
|
+
[tool.pytest.ini_options]
|
|
35
|
+
testpaths = ["tests"]
|
|
36
|
+
addopts = "-q"
|
attnview-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""A focused, terminal-first viewer for transformer attention patterns."""
|
|
2
|
+
|
|
3
|
+
from .api import render_html, show
|
|
4
|
+
from .model import AttentionPattern, AttentionShapeError
|
|
5
|
+
|
|
6
|
+
__all__ = ["AttentionPattern", "AttentionShapeError", "render_html", "show"]
|
|
7
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Public API for terminal and standalone HTML attention visualizations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any, Sequence
|
|
7
|
+
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
|
|
10
|
+
from .html import build_html
|
|
11
|
+
from .model import AttentionPattern
|
|
12
|
+
from .terminal import build_terminal
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def show(
|
|
16
|
+
attention: Any,
|
|
17
|
+
*,
|
|
18
|
+
tokens: Sequence[str] | None = None,
|
|
19
|
+
query_tokens: Sequence[str] | None = None,
|
|
20
|
+
key_tokens: Sequence[str] | None = None,
|
|
21
|
+
head: int = 0,
|
|
22
|
+
output: str | Path | None = None,
|
|
23
|
+
top_k: int = 3,
|
|
24
|
+
title: str = "Attention Explorer",
|
|
25
|
+
console: Console | None = None,
|
|
26
|
+
) -> Path | None:
|
|
27
|
+
"""Show `[heads, queries, keys]` attention in the terminal or write HTML.
|
|
28
|
+
|
|
29
|
+
`output=None` prints a Rich-rendered view. Pass a path ending in `.html` to
|
|
30
|
+
write a portable interactive report. The return value is that path for HTML
|
|
31
|
+
output and `None` for terminal output.
|
|
32
|
+
"""
|
|
33
|
+
pattern = AttentionPattern.from_data(
|
|
34
|
+
attention, tokens=tokens, query_tokens=query_tokens, key_tokens=key_tokens
|
|
35
|
+
)
|
|
36
|
+
pattern.head(head)
|
|
37
|
+
if top_k < 1:
|
|
38
|
+
raise ValueError("top_k must be at least 1")
|
|
39
|
+
if output is None:
|
|
40
|
+
(console or Console()).print(build_terminal(pattern, head=head, top_k=top_k, title=title))
|
|
41
|
+
return None
|
|
42
|
+
path = Path(output)
|
|
43
|
+
if path.suffix.lower() != ".html":
|
|
44
|
+
raise ValueError("output must be an .html path")
|
|
45
|
+
path.write_text(build_html(pattern, head=head, top_k=top_k, title=title), encoding="utf-8")
|
|
46
|
+
return path
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def render_html(
|
|
50
|
+
attention: Any,
|
|
51
|
+
*,
|
|
52
|
+
tokens: Sequence[str] | None = None,
|
|
53
|
+
query_tokens: Sequence[str] | None = None,
|
|
54
|
+
key_tokens: Sequence[str] | None = None,
|
|
55
|
+
head: int = 0,
|
|
56
|
+
top_k: int = 3,
|
|
57
|
+
title: str = "Attention Explorer",
|
|
58
|
+
) -> str:
|
|
59
|
+
"""Return a self-contained interactive HTML report as a string."""
|
|
60
|
+
pattern = AttentionPattern.from_data(
|
|
61
|
+
attention, tokens=tokens, query_tokens=query_tokens, key_tokens=key_tokens
|
|
62
|
+
)
|
|
63
|
+
pattern.head(head)
|
|
64
|
+
if top_k < 1:
|
|
65
|
+
raise ValueError("top_k must be at least 1")
|
|
66
|
+
return build_html(pattern, head=head, top_k=top_k, title=title)
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""Standalone, dependency-free HTML report generation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import html
|
|
6
|
+
import json
|
|
7
|
+
|
|
8
|
+
from .model import AttentionPattern
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
_DOCUMENT = '''<!doctype html>
|
|
12
|
+
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
|
|
13
|
+
<title>__TITLE__</title><style>
|
|
14
|
+
:root{--ink:#0b141d;--surface:#121f2b;--line:#294253;--muted:#91a9b8;--paper:#e8f0f3;--cyan:#51c5c1;--amber:#f2b84b}
|
|
15
|
+
*{box-sizing:border-box}body{margin:0;background:var(--ink);color:var(--paper);font-family:ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif}
|
|
16
|
+
main{max-width:1240px;margin:auto;padding:42px 28px 64px}header{display:flex;justify-content:space-between;align-items:end;border-bottom:1px solid var(--line);padding-bottom:22px;margin-bottom:28px;gap:20px}
|
|
17
|
+
h1{font-size:clamp(1.7rem,4vw,2.8rem);letter-spacing:-.045em;margin:0}header p{margin:7px 0 0;color:var(--muted)}select{background:var(--surface);color:var(--paper);border:1px solid var(--line);border-radius:7px;padding:10px 12px;font:inherit}
|
|
18
|
+
.workspace{display:grid;grid-template-columns:minmax(0,1fr) 270px;gap:22px}.plot,.detail,.heads{background:var(--surface);border:1px solid var(--line);border-radius:12px}.plot{padding:22px;overflow:auto}.axis{color:var(--muted);font-size:.82rem;margin-bottom:10px}
|
|
19
|
+
.heatmap{display:grid;gap:2px;min-width:500px;width:min(100%,720px)}.cell{aspect-ratio:1;border:0;border-radius:2px;cursor:pointer;min-width:12px}.cell:hover,.cell.selected{outline:2px solid var(--paper);outline-offset:1px;z-index:1}.labels{display:grid;gap:2px;min-width:396px;width:calc(min(100% - 104px,616px));margin-left:104px;margin-bottom:5px}.label{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;text-align:center;color:#bbcad3;font:12px ui-monospace,SFMono-Regular,Menlo,monospace}
|
|
20
|
+
.side{display:grid;gap:22px;align-content:start}.detail{padding:20px}.detail h2,.heads h2{font-size:.95rem;margin:0 0 16px;color:#c9d8df}.pair{font:600 1.15rem ui-monospace,SFMono-Regular,Menlo,monospace;color:var(--amber);margin:6px 0 16px}.metric{display:flex;justify-content:space-between;padding:8px 0;border-top:1px solid var(--line);color:var(--muted)}.metric strong{color:var(--paper);font-weight:600}
|
|
21
|
+
.heads{padding:18px;margin-top:22px}.head-list{display:flex;gap:10px;overflow:auto}.mini{min-width:88px;background:#0d1923;border:1px solid #254153;border-radius:7px;padding:7px;cursor:pointer;color:var(--muted);font-size:.77rem}.mini.active{border-color:var(--cyan);color:var(--paper)}.mini-grid{display:grid;grid-template-columns:repeat(6,1fr);gap:1px;height:48px;margin-bottom:6px}.mini-cell{display:block;border-radius:1px}
|
|
22
|
+
footer{color:var(--muted);font-size:.84rem;margin-top:25px}@media(max-width:780px){main{padding:24px 14px}header{align-items:start;flex-direction:column}.workspace{grid-template-columns:1fr}.side{grid-template-columns:1fr 1fr}.heads{margin-top:16px}}@media(max-width:520px){.side{grid-template-columns:1fr}}
|
|
23
|
+
</style></head><body><main><header><div><h1>__TITLE__</h1><p>Attention as a map: query rows, key columns.</p></div><label>Head <select id="head"></select></label></header><section class="workspace"><div class="plot"><div class="axis">K · attended-to token →</div><div id="key-labels" class="labels"></div><div class="axis">Q · attending token ↓</div><div id="heatmap" class="heatmap" role="grid" aria-label="Attention heatmap"></div></div><aside class="side"><section class="detail"><h2>Selected relation</h2><div id="pair" class="pair"></div><div class="metric"><span>Attention</span><strong id="weight"></strong></div><div class="metric"><span>Rank within query</span><strong id="rank"></strong></div><div class="metric"><span>Top keys</span><strong id="top"></strong></div></section></aside></section><section class="heads"><h2>All heads</h2><div id="head-list" class="head-list"></div></section><footer>Amber indicates high attention; teal indicates moderate attention. Click a cell to inspect it.</footer></main>
|
|
24
|
+
<script>
|
|
25
|
+
const data = __DATA__;
|
|
26
|
+
let selected = {h: data.head, q: 0, k: 0};
|
|
27
|
+
const $ = id => document.getElementById(id);
|
|
28
|
+
const max = array => Math.max(...array.flat(2));
|
|
29
|
+
function color(value, maximum) {
|
|
30
|
+
const ratio = Math.max(0, Math.min(1, value / (maximum || 1)));
|
|
31
|
+
const stops = [[11,20,32],[20,56,75],[46,146,151],[242,184,75]];
|
|
32
|
+
const position = ratio * (stops.length - 1), index = Math.min(Math.floor(position), stops.length - 2), mix = position - index;
|
|
33
|
+
const a = stops[index], b = stops[index + 1];
|
|
34
|
+
return `rgb(${Math.round(a[0]*(1-mix)+b[0]*mix)},${Math.round(a[1]*(1-mix)+b[1]*mix)},${Math.round(a[2]*(1-mix)+b[2]*mix)})`;
|
|
35
|
+
}
|
|
36
|
+
function drawHeads(maximum) {
|
|
37
|
+
const list = $('head-list'); list.innerHTML = '';
|
|
38
|
+
data.heads.forEach((rows, head) => {
|
|
39
|
+
const button = document.createElement('button'); button.className = 'mini' + (head === selected.h ? ' active' : '');
|
|
40
|
+
const values = rows.flat(), sample = Array.from({length: 36}, (_, index) => values[Math.floor(index * values.length / 36)]);
|
|
41
|
+
button.innerHTML = `<div class="mini-grid">${sample.map(value => `<i class="mini-cell" style="background:${color(value, maximum)}"></i>`).join('')}</div>Head ${head + 1}`;
|
|
42
|
+
button.onclick = () => { selected = {h: head, q: 0, k: 0}; draw(); }; list.append(button);
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
function draw() {
|
|
46
|
+
const rows = data.heads[selected.h], maximum = max(data.heads), count = data.keys.length;
|
|
47
|
+
$('head').value = selected.h;
|
|
48
|
+
const labels = $('key-labels'); labels.style.gridTemplateColumns = `repeat(${count}, 1fr)`;
|
|
49
|
+
labels.innerHTML = data.keys.map(key => `<div class="label" title="${key}">${key}</div>`).join('');
|
|
50
|
+
const grid = $('heatmap'); grid.style.gridTemplateColumns = `96px repeat(${count}, 1fr)`; grid.innerHTML = '';
|
|
51
|
+
rows.forEach((row, query) => {
|
|
52
|
+
const label = document.createElement('div'); label.className = 'label'; label.style.textAlign = 'right'; label.style.paddingRight = '10px'; label.textContent = data.queries[query]; grid.append(label);
|
|
53
|
+
row.forEach((value, key) => {
|
|
54
|
+
const cell = document.createElement('button'); cell.className = 'cell' + (query === selected.q && key === selected.k ? ' selected' : '');
|
|
55
|
+
cell.style.background = color(value, maximum); cell.title = `${data.queries[query]} → ${data.keys[key]}: ${value.toFixed(4)}`;
|
|
56
|
+
cell.onclick = () => { selected.q = query; selected.k = key; draw(); }; grid.append(cell);
|
|
57
|
+
});
|
|
58
|
+
});
|
|
59
|
+
const values = rows[selected.q], value = values[selected.k], rank = values.filter(item => item > value).length + 1;
|
|
60
|
+
const top = [...values.keys()].sort((a,b) => values[b] - values[a]).slice(0, data.topK).map(index => data.keys[index]).join(', ');
|
|
61
|
+
$('pair').textContent = `${data.queries[selected.q]} → ${data.keys[selected.k]}`; $('weight').textContent = value.toFixed(4); $('rank').textContent = '#' + rank; $('top').textContent = top;
|
|
62
|
+
drawHeads(maximum);
|
|
63
|
+
}
|
|
64
|
+
data.heads.forEach((_, index) => $('head').insertAdjacentHTML('beforeend', `<option value="${index}">Head ${index + 1}</option>`));
|
|
65
|
+
$('head').onchange = event => { selected = {h: +event.target.value, q: 0, k: 0}; draw(); };
|
|
66
|
+
draw();
|
|
67
|
+
</script></body></html>'''
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def build_html(pattern: AttentionPattern, *, head: int, top_k: int, title: str) -> str:
|
|
71
|
+
payload = json.dumps(
|
|
72
|
+
{"heads": pattern.values, "queries": pattern.query_tokens, "keys": pattern.key_tokens, "head": head, "topK": top_k},
|
|
73
|
+
ensure_ascii=False,
|
|
74
|
+
).replace("</", "<\\/")
|
|
75
|
+
return _DOCUMENT.replace("__DATA__", payload).replace("__TITLE__", html.escape(title))
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Input normalization and validation for attention patterns."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from math import isfinite
|
|
7
|
+
from typing import Any, Sequence
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class AttentionShapeError(ValueError):
|
|
11
|
+
"""Raised when an object cannot represent a 3D attention pattern."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _to_nested_list(value: Any) -> list[Any]:
|
|
15
|
+
"""Accept NumPy/PyTorch-like values without making either a dependency."""
|
|
16
|
+
if hasattr(value, "detach"):
|
|
17
|
+
value = value.detach()
|
|
18
|
+
if hasattr(value, "cpu"):
|
|
19
|
+
value = value.cpu()
|
|
20
|
+
if hasattr(value, "tolist"):
|
|
21
|
+
value = value.tolist()
|
|
22
|
+
if not isinstance(value, (list, tuple)):
|
|
23
|
+
raise AttentionShapeError("attention must be a 3D sequence or array")
|
|
24
|
+
return list(value)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _float(value: Any) -> float:
|
|
28
|
+
try:
|
|
29
|
+
number = float(value)
|
|
30
|
+
except (TypeError, ValueError) as error:
|
|
31
|
+
raise AttentionShapeError("attention values must be numeric") from error
|
|
32
|
+
if not isfinite(number):
|
|
33
|
+
raise AttentionShapeError("attention values must be finite")
|
|
34
|
+
return number
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _labels(values: Sequence[str] | None, length: int, prefix: str) -> tuple[str, ...]:
|
|
38
|
+
if values is None:
|
|
39
|
+
return tuple(f"{prefix}{index}" for index in range(length))
|
|
40
|
+
labels = tuple(str(value) for value in values)
|
|
41
|
+
if len(labels) != length:
|
|
42
|
+
raise AttentionShapeError(
|
|
43
|
+
f"expected {length} {prefix.lower()} token labels, received {len(labels)}"
|
|
44
|
+
)
|
|
45
|
+
return labels
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class AttentionPattern:
|
|
50
|
+
"""A validated `[heads, queries, keys]` attention tensor."""
|
|
51
|
+
|
|
52
|
+
values: tuple[tuple[tuple[float, ...], ...], ...]
|
|
53
|
+
query_tokens: tuple[str, ...]
|
|
54
|
+
key_tokens: tuple[str, ...]
|
|
55
|
+
|
|
56
|
+
@classmethod
|
|
57
|
+
def from_data(
|
|
58
|
+
cls,
|
|
59
|
+
attention: Any,
|
|
60
|
+
*,
|
|
61
|
+
tokens: Sequence[str] | None = None,
|
|
62
|
+
query_tokens: Sequence[str] | None = None,
|
|
63
|
+
key_tokens: Sequence[str] | None = None,
|
|
64
|
+
) -> "AttentionPattern":
|
|
65
|
+
if tokens is not None and (query_tokens is not None or key_tokens is not None):
|
|
66
|
+
raise AttentionShapeError("use `tokens` or explicit query/key tokens, not both")
|
|
67
|
+
data = _to_nested_list(attention)
|
|
68
|
+
if not data:
|
|
69
|
+
raise AttentionShapeError("attention must contain at least one head")
|
|
70
|
+
heads: list[tuple[tuple[float, ...], ...]] = []
|
|
71
|
+
query_length: int | None = None
|
|
72
|
+
key_length: int | None = None
|
|
73
|
+
for head in data:
|
|
74
|
+
if not isinstance(head, (list, tuple)) or not head:
|
|
75
|
+
raise AttentionShapeError("every head must contain at least one query row")
|
|
76
|
+
rows: list[tuple[float, ...]] = []
|
|
77
|
+
for row in head:
|
|
78
|
+
if not isinstance(row, (list, tuple)) or not row:
|
|
79
|
+
raise AttentionShapeError("every query row must contain at least one key")
|
|
80
|
+
values = tuple(_float(item) for item in row)
|
|
81
|
+
if key_length is None:
|
|
82
|
+
key_length = len(values)
|
|
83
|
+
elif len(values) != key_length:
|
|
84
|
+
raise AttentionShapeError("all query rows must have the same key length")
|
|
85
|
+
rows.append(values)
|
|
86
|
+
if query_length is None:
|
|
87
|
+
query_length = len(rows)
|
|
88
|
+
elif len(rows) != query_length:
|
|
89
|
+
raise AttentionShapeError("all heads must have the same query length")
|
|
90
|
+
heads.append(tuple(rows))
|
|
91
|
+
assert query_length is not None and key_length is not None
|
|
92
|
+
if tokens is not None and query_length != key_length:
|
|
93
|
+
raise AttentionShapeError("`tokens` is only valid when query and key lengths match")
|
|
94
|
+
return cls(
|
|
95
|
+
values=tuple(heads),
|
|
96
|
+
query_tokens=_labels(tokens if tokens is not None else query_tokens, query_length, "Q"),
|
|
97
|
+
key_tokens=_labels(tokens if tokens is not None else key_tokens, key_length, "K"),
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
@property
|
|
101
|
+
def head_count(self) -> int:
|
|
102
|
+
return len(self.values)
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def query_length(self) -> int:
|
|
106
|
+
return len(self.values[0])
|
|
107
|
+
|
|
108
|
+
@property
|
|
109
|
+
def key_length(self) -> int:
|
|
110
|
+
return len(self.values[0][0])
|
|
111
|
+
|
|
112
|
+
def head(self, index: int) -> tuple[tuple[float, ...], ...]:
|
|
113
|
+
if not 0 <= index < self.head_count:
|
|
114
|
+
raise IndexError(f"head must be in [0, {self.head_count - 1}], received {index}")
|
|
115
|
+
return self.values[index]
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Rich terminal rendering."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from rich import box
|
|
6
|
+
from rich.columns import Columns
|
|
7
|
+
from rich.panel import Panel
|
|
8
|
+
from rich.table import Table
|
|
9
|
+
from rich.text import Text
|
|
10
|
+
|
|
11
|
+
from .model import AttentionPattern
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _colour(value: float, maximum: float) -> str:
|
|
15
|
+
ratio = 0 if maximum <= 0 else max(0.0, min(1.0, value / maximum))
|
|
16
|
+
# Ink -> deep blue -> teal -> amber. Designed for dark terminals.
|
|
17
|
+
stops = ((11, 20, 32), (20, 56, 75), (46, 146, 151), (242, 184, 75))
|
|
18
|
+
position = ratio * (len(stops) - 1)
|
|
19
|
+
left = min(int(position), len(stops) - 2)
|
|
20
|
+
mix = position - left
|
|
21
|
+
rgb = tuple(round(stops[left][i] * (1 - mix) + stops[left + 1][i] * mix) for i in range(3))
|
|
22
|
+
return "#%02x%02x%02x" % rgb
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _short(label: str, width: int = 9) -> str:
|
|
26
|
+
return label if len(label) <= width else label[: width - 1] + "…"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def build_terminal(pattern: AttentionPattern, *, head: int, top_k: int, title: str) -> Panel:
|
|
30
|
+
matrix = pattern.head(head)
|
|
31
|
+
maximum = max(value for row in matrix for value in row)
|
|
32
|
+
table = Table(box=box.SIMPLE_HEAVY, pad_edge=False, show_header=True, header_style="bold #b9c9d7")
|
|
33
|
+
table.add_column("Q ↓ / K →", style="#b9c9d7", no_wrap=True)
|
|
34
|
+
for token in pattern.key_tokens:
|
|
35
|
+
table.add_column(_short(token), justify="center", width=3, no_wrap=True)
|
|
36
|
+
for token, row in zip(pattern.query_tokens, matrix):
|
|
37
|
+
cells = [
|
|
38
|
+
Text("██", style=f"{_colour(value, maximum)} on {_colour(value, maximum)}")
|
|
39
|
+
for value in row
|
|
40
|
+
]
|
|
41
|
+
table.add_row(_short(token), *cells)
|
|
42
|
+
|
|
43
|
+
strongest = max(range(pattern.query_length), key=lambda index: max(matrix[index]))
|
|
44
|
+
ranking = sorted(enumerate(matrix[strongest]), key=lambda item: item[1], reverse=True)[:top_k]
|
|
45
|
+
summary = Table.grid(padding=(0, 1))
|
|
46
|
+
summary.add_column(style="bold #f2b84b")
|
|
47
|
+
summary.add_column(justify="right", style="#e6edf3")
|
|
48
|
+
for index, value in ranking:
|
|
49
|
+
summary.add_row(_short(pattern.key_tokens[index], 16), f"{value:.3f}")
|
|
50
|
+
sidebar = Panel(
|
|
51
|
+
summary,
|
|
52
|
+
title=f"Top keys for {_short(pattern.query_tokens[strongest], 16)}",
|
|
53
|
+
subtitle="strongest query",
|
|
54
|
+
border_style="#27626a",
|
|
55
|
+
padding=(1, 1),
|
|
56
|
+
)
|
|
57
|
+
layout = Columns([table, sidebar], expand=False, padding=(2, 0))
|
|
58
|
+
caption = "Rows are queries; columns are keys. Use `head=` to inspect another head."
|
|
59
|
+
return Panel(
|
|
60
|
+
layout,
|
|
61
|
+
title=f"[bold #e6edf3]{title}[/] [#6f93a8]head {head + 1}/{pattern.head_count} · {pattern.query_length} Q × {pattern.key_length} K[/]",
|
|
62
|
+
subtitle=f"[#76d4d4]{caption}[/]",
|
|
63
|
+
border_style="#315166",
|
|
64
|
+
padding=(1, 1),
|
|
65
|
+
)
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: attnview
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Beautiful terminal and HTML views for transformer attention patterns.
|
|
5
|
+
Author-email: Pedro Gustavo <pedrogustavosilva3060@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/pegruk/attnview
|
|
8
|
+
Project-URL: Issues, https://github.com/pegruk/attnview/issues
|
|
9
|
+
Keywords: attention,transformers,interpretability,visualization,terminal
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: rich>=13.7
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
22
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
23
|
+
Requires-Dist: twine>=5.0; extra == "dev"
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# attnview
|
|
27
|
+
|
|
28
|
+
`attnview` is a small, terminal-first viewer for transformer attention patterns.
|
|
29
|
+
It accepts a single attention layer shaped `[n_heads, query_length, key_length]`,
|
|
30
|
+
keeps query and key axes explicit, and renders either a readable terminal heatmap
|
|
31
|
+
or a self-contained interactive HTML report.
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install attnview
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
import attnview
|
|
39
|
+
|
|
40
|
+
# `attention` can be a NumPy array, a PyTorch tensor, or nested Python sequences.
|
|
41
|
+
attnview.show(attention, tokens=["[BOS]", "The", "cat", "sat", "."])
|
|
42
|
+
|
|
43
|
+
# Write a standalone report that can be opened or shared without a server.
|
|
44
|
+
attnview.show(attention, tokens=tokens, output="attention.html")
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Reading the view
|
|
48
|
+
|
|
49
|
+
- **Rows are queries**: the token doing the attending.
|
|
50
|
+
- **Columns are keys**: the token being attended to.
|
|
51
|
+
- The selected head is rendered in the main heatmap; the HTML report also offers
|
|
52
|
+
one-click head selection through miniature maps.
|
|
53
|
+
- Amber marks high attention, teal marks moderate attention, and the darkest cells
|
|
54
|
+
are near zero. This deliberately avoids a rainbow palette that makes magnitude
|
|
55
|
+
difficult to read.
|
|
56
|
+
|
|
57
|
+
## API
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
attnview.show(
|
|
61
|
+
attention,
|
|
62
|
+
tokens=None,
|
|
63
|
+
query_tokens=None,
|
|
64
|
+
key_tokens=None,
|
|
65
|
+
head=0,
|
|
66
|
+
output=None,
|
|
67
|
+
top_k=3,
|
|
68
|
+
title="Attention Explorer",
|
|
69
|
+
)
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Pass `tokens` for self-attention. For cross-attention, pass `query_tokens` and
|
|
73
|
+
`key_tokens` independently. With `output=None` (the default), `show` prints to
|
|
74
|
+
the current terminal. With an `.html` path, it writes a standalone HTML document
|
|
75
|
+
and returns its `Path`.
|
|
76
|
+
|
|
77
|
+
`render_html(...)` returns the standalone HTML string when embedding it elsewhere.
|
|
78
|
+
|
|
79
|
+
## Requirements
|
|
80
|
+
|
|
81
|
+
Python 3.10+ and [Rich](https://github.com/Textualize/rich). NumPy and PyTorch are
|
|
82
|
+
not required: if they are installed, their arrays/tensors are accepted directly.
|
|
83
|
+
|
|
84
|
+
## Development
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
python -m pip install -e '.[dev]'
|
|
88
|
+
python -m pytest
|
|
89
|
+
python -m build
|
|
90
|
+
python -m twine check dist/*
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## License
|
|
94
|
+
|
|
95
|
+
MIT
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/attnview/__init__.py
|
|
5
|
+
src/attnview/api.py
|
|
6
|
+
src/attnview/html.py
|
|
7
|
+
src/attnview/model.py
|
|
8
|
+
src/attnview/terminal.py
|
|
9
|
+
src/attnview.egg-info/PKG-INFO
|
|
10
|
+
src/attnview.egg-info/SOURCES.txt
|
|
11
|
+
src/attnview.egg-info/dependency_links.txt
|
|
12
|
+
src/attnview.egg-info/requires.txt
|
|
13
|
+
src/attnview.egg-info/top_level.txt
|
|
14
|
+
tests/test_attnview.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
attnview
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
from rich.console import Console
|
|
5
|
+
|
|
6
|
+
import attnview
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
ATTENTION = [
|
|
10
|
+
[[0.05, 0.85, 0.10], [0.20, 0.30, 0.50], [0.70, 0.20, 0.10]],
|
|
11
|
+
[[0.30, 0.20, 0.50], [0.60, 0.30, 0.10], [0.15, 0.70, 0.15]],
|
|
12
|
+
]
|
|
13
|
+
TOKENS = ["The", "cat", "sat"]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_pattern_accepts_self_attention_tokens():
|
|
17
|
+
pattern = attnview.AttentionPattern.from_data(ATTENTION, tokens=TOKENS)
|
|
18
|
+
assert pattern.head_count == 2
|
|
19
|
+
assert pattern.query_length == pattern.key_length == 3
|
|
20
|
+
assert pattern.query_tokens == pattern.key_tokens == tuple(TOKENS)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_pattern_accepts_cross_attention_labels():
|
|
24
|
+
pattern = attnview.AttentionPattern.from_data(
|
|
25
|
+
[[[0.2, 0.8], [0.7, 0.3]]], query_tokens=["decode", "next"], key_tokens=["a", "b"]
|
|
26
|
+
)
|
|
27
|
+
assert pattern.query_tokens == ("decode", "next")
|
|
28
|
+
assert pattern.key_tokens == ("a", "b")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@pytest.mark.parametrize(
|
|
32
|
+
"data, message",
|
|
33
|
+
[([], "at least one head"), ([[[1.0], [0.3, 0.7]]], "same key length"), ([[[float('nan')]]], "finite")],
|
|
34
|
+
)
|
|
35
|
+
def test_invalid_shapes_are_explained(data, message):
|
|
36
|
+
with pytest.raises(attnview.AttentionShapeError, match=message):
|
|
37
|
+
attnview.AttentionPattern.from_data(data)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_terminal_show_renders_tokens_and_axis_labels():
|
|
41
|
+
console = Console(record=True, width=120, force_terminal=False)
|
|
42
|
+
result = attnview.show(ATTENTION, tokens=TOKENS, head=1, console=console)
|
|
43
|
+
output = console.export_text()
|
|
44
|
+
assert result is None
|
|
45
|
+
assert "Attention Explorer" in output
|
|
46
|
+
assert "The" in output and "cat" in output
|
|
47
|
+
assert "Top keys" in output
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_html_output_is_standalone_and_interactive(tmp_path: Path):
|
|
51
|
+
target = tmp_path / "report.html"
|
|
52
|
+
returned = attnview.show(ATTENTION, tokens=TOKENS, output=target, title="My map")
|
|
53
|
+
report = target.read_text(encoding="utf-8")
|
|
54
|
+
assert returned == target
|
|
55
|
+
assert "My map" in report
|
|
56
|
+
assert "Selected relation" in report
|
|
57
|
+
assert "data.heads.forEach" in report
|
|
58
|
+
assert "Head ${index + 1}" in report
|
|
59
|
+
assert "The" in report
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def test_render_html_rejects_non_positive_top_k():
|
|
63
|
+
with pytest.raises(ValueError, match="at least 1"):
|
|
64
|
+
attnview.render_html(ATTENTION, tokens=TOKENS, top_k=0)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_output_requires_html_suffix(tmp_path: Path):
|
|
68
|
+
with pytest.raises(ValueError, match=".html"):
|
|
69
|
+
attnview.show(ATTENTION, tokens=TOKENS, output=tmp_path / "report.txt")
|