tensi-shapes 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tensi_shapes-0.3.0/LICENSE +9 -0
- tensi_shapes-0.3.0/PKG-INFO +119 -0
- tensi_shapes-0.3.0/README.md +87 -0
- tensi_shapes-0.3.0/pyproject.toml +60 -0
- tensi_shapes-0.3.0/setup.cfg +4 -0
- tensi_shapes-0.3.0/tensi/__init__.py +18 -0
- tensi_shapes-0.3.0/tensi/__main__.py +3 -0
- tensi_shapes-0.3.0/tensi/cli.py +298 -0
- tensi_shapes-0.3.0/tensi/doctor.py +226 -0
- tensi_shapes-0.3.0/tensi/mapping.py +159 -0
- tensi_shapes-0.3.0/tensi/peek.py +155 -0
- tensi_shapes-0.3.0/tensi/rewrite.py +1933 -0
- tensi_shapes-0.3.0/tensi/scratch.py +43 -0
- tensi_shapes-0.3.0/tensi/seeds.py +1533 -0
- tensi_shapes-0.3.0/tensi/server.py +966 -0
- tensi_shapes-0.3.0/tensi/stubs.py +530 -0
- tensi_shapes-0.3.0/tensi_shapes.egg-info/PKG-INFO +119 -0
- tensi_shapes-0.3.0/tensi_shapes.egg-info/SOURCES.txt +24 -0
- tensi_shapes-0.3.0/tensi_shapes.egg-info/dependency_links.txt +1 -0
- tensi_shapes-0.3.0/tensi_shapes.egg-info/entry_points.txt +2 -0
- tensi_shapes-0.3.0/tensi_shapes.egg-info/requires.txt +9 -0
- tensi_shapes-0.3.0/tensi_shapes.egg-info/top_level.txt +1 -0
- tensi_shapes-0.3.0/tests/test_cli.py +266 -0
- tensi_shapes-0.3.0/tests/test_lsp.py +462 -0
- tensi_shapes-0.3.0/tests/test_rewrite.py +585 -0
- tensi_shapes-0.3.0/tests/test_stubs.py +112 -0
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Tristan Farran
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
6
|
+
|
|
7
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
8
|
+
|
|
9
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tensi-shapes
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Named tensor shapes for unannotated PyTorch: hovers, shape errors and a CLI, built on Pyrefly.
|
|
5
|
+
Author: Tristan Farran
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/tristan-farran/tensi
|
|
8
|
+
Project-URL: Issues, https://github.com/tristan-farran/tensi/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/tristan-farran/tensi/blob/main/CHANGELOG.md
|
|
10
|
+
Keywords: pytorch,tensor,shapes,type-checking,pyrefly,jaxtyping,einops,lsp
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
20
|
+
Requires-Python: >=3.12
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: pyrefly==1.5.0
|
|
24
|
+
Requires-Dist: pyrefly-torch-stubs==1.5.0
|
|
25
|
+
Requires-Dist: pyrefly-einops-stubs==1.5.0
|
|
26
|
+
Requires-Dist: parso>=0.8.4
|
|
27
|
+
Provides-Extra: test
|
|
28
|
+
Requires-Dist: pytest>=8; extra == "test"
|
|
29
|
+
Requires-Dist: jaxtyping; extra == "test"
|
|
30
|
+
Requires-Dist: einops; extra == "test"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
<p align="center">
|
|
34
|
+
<img src="https://raw.githubusercontent.com/tristan-farran/tensi/main/brand/tensi-logo.svg" alt="tensi" width="440">
|
|
35
|
+
</p>
|
|
36
|
+
|
|
37
|
+
<p align="center">
|
|
38
|
+
<b>tensi sees your tensors.</b><br>
|
|
39
|
+
Hover any PyTorch tensor and see its shape, with every dimension named.
|
|
40
|
+
</p>
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
def forward(self, x):
|
|
46
|
+
B, T, C = x.size()
|
|
47
|
+
q, k, v = self.c_attn(x).split(self.n_embd, dim=2)
|
|
48
|
+
k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
|
|
49
|
+
# hover k → Tensor[[b, n_head, t, n_embd // n_head]]
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
## Install
|
|
54
|
+
|
|
55
|
+
**VS Code:** install **tensi** from the Marketplace. It follows each project's interpreter (the Python
|
|
56
|
+
extension's choice, else `.venv`), so switching projects or running `uv sync` needs no setup. If tensi is
|
|
57
|
+
missing it offers `uv add --dev tensi-shapes` for the project, or `uv tool install tensi-shapes` once for
|
|
58
|
+
all projects. It runs beside Pylance and only adds shapes.
|
|
59
|
+
|
|
60
|
+
**Command line, CI, other editors:**
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
uv add --dev tensi-shapes # or: pip install tensi-shapes, into the project's environment
|
|
64
|
+
uv tool install tensi-shapes # or once for all projects; it reads torch from each project's .venv
|
|
65
|
+
tensi doctor # checks Pyrefly, the stubs and the interpreter Pyrefly picks, with one real probe
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
The package is `tensi-shapes` on PyPI; the command and the import are `tensi`. It brings Pyrefly 1.5.0
|
|
69
|
+
and its torch shape stubs; your project needs `torch` installed.
|
|
70
|
+
|
|
71
|
+
Any editor that speaks LSP can run `tensi lsp` as a language server for Python files. Python 3.12+;
|
|
72
|
+
`rearrange` and friends are shape-aware on Python 3.12+ (Pyrefly's einops stubs need it).
|
|
73
|
+
|
|
74
|
+
## Try it
|
|
75
|
+
|
|
76
|
+
```console
|
|
77
|
+
$ tensi peek model.py
|
|
78
|
+
tensi peek v0.3.0 model.py 2.0s
|
|
79
|
+
|
|
80
|
+
CausalSelfAttention.forward
|
|
81
|
+
56 q, k, v [b, t, n_embd]
|
|
82
|
+
57 k [b, n_head, t, n_embd // n_head]
|
|
83
|
+
67 att [b, n_head, t, t]
|
|
84
|
+
72 y [b, t, n_embd]
|
|
85
|
+
MLP.forward
|
|
86
|
+
88 x [b, t, 4 * n_embd]
|
|
87
|
+
GPT.forward
|
|
88
|
+
177 tok_emb [b, t, n_embd]
|
|
89
|
+
178 pos_emb [t, n_embd]
|
|
90
|
+
186 logits [b, t, vocab_size]
|
|
91
|
+
GPT.generate
|
|
92
|
+
321 v, _ [b, int]
|
|
93
|
+
326 idx_next [b, 1]
|
|
94
|
+
|
|
95
|
+
37 tensors, 32 fully named
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
```console
|
|
99
|
+
$ tensi check model.py
|
|
100
|
+
tensi check v0.3.0 model.py 2.0s
|
|
101
|
+
ok no shape errors
|
|
102
|
+
|
|
103
|
+
$ tensi check bad.py
|
|
104
|
+
tensi check v0.3.0 bad.py 819ms
|
|
105
|
+
x bad.py:17:9 Argument `Tensor[[b, 7, v]]` is not assignable to parameter `x` with type `Tensor[[b, 7, d]]` in function `Block.forward`
|
|
106
|
+
x bad.py:18:9 `@` is not supported between `Tensor[[b, d]]` and `Tensor[[v, 3]]`
|
|
107
|
+
|
|
108
|
+
2 shape errors
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
`tensi check src/` checks every Python file under a folder and exits 1 when it finds a shape error, so it
|
|
112
|
+
can gate CI.
|
|
113
|
+
|
|
114
|
+
## Contributing
|
|
115
|
+
|
|
116
|
+
See [CONTRIBUTING.md](https://github.com/tristan-farran/tensi/blob/main/CONTRIBUTING.md) for the development setup, tests and releases.
|
|
117
|
+
|
|
118
|
+
<sub>tensi is built on Pyrefly (MIT) and its torch stubs. The mascot is a small tensor that is very good at
|
|
119
|
+
looking.</sub>
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/tristan-farran/tensi/main/brand/tensi-logo.svg" alt="tensi" width="440">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<p align="center">
|
|
6
|
+
<b>tensi sees your tensors.</b><br>
|
|
7
|
+
Hover any PyTorch tensor and see its shape, with every dimension named.
|
|
8
|
+
</p>
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
```python
|
|
13
|
+
def forward(self, x):
|
|
14
|
+
B, T, C = x.size()
|
|
15
|
+
q, k, v = self.c_attn(x).split(self.n_embd, dim=2)
|
|
16
|
+
k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
|
|
17
|
+
# hover k → Tensor[[b, n_head, t, n_embd // n_head]]
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
## Install
|
|
22
|
+
|
|
23
|
+
**VS Code:** install **tensi** from the Marketplace. It follows each project's interpreter (the Python
|
|
24
|
+
extension's choice, else `.venv`), so switching projects or running `uv sync` needs no setup. If tensi is
|
|
25
|
+
missing it offers `uv add --dev tensi-shapes` for the project, or `uv tool install tensi-shapes` once for
|
|
26
|
+
all projects. It runs beside Pylance and only adds shapes.
|
|
27
|
+
|
|
28
|
+
**Command line, CI, other editors:**
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
uv add --dev tensi-shapes # or: pip install tensi-shapes, into the project's environment
|
|
32
|
+
uv tool install tensi-shapes # or once for all projects; it reads torch from each project's .venv
|
|
33
|
+
tensi doctor # checks Pyrefly, the stubs and the interpreter Pyrefly picks, with one real probe
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
The package is `tensi-shapes` on PyPI; the command and the import are `tensi`. It brings Pyrefly 1.5.0
|
|
37
|
+
and its torch shape stubs; your project needs `torch` installed.
|
|
38
|
+
|
|
39
|
+
Any editor that speaks LSP can run `tensi lsp` as a language server for Python files. Python 3.12+;
|
|
40
|
+
`rearrange` and friends are shape-aware on Python 3.12+ (Pyrefly's einops stubs need it).
|
|
41
|
+
|
|
42
|
+
## Try it
|
|
43
|
+
|
|
44
|
+
```console
|
|
45
|
+
$ tensi peek model.py
|
|
46
|
+
tensi peek v0.3.0 model.py 2.0s
|
|
47
|
+
|
|
48
|
+
CausalSelfAttention.forward
|
|
49
|
+
56 q, k, v [b, t, n_embd]
|
|
50
|
+
57 k [b, n_head, t, n_embd // n_head]
|
|
51
|
+
67 att [b, n_head, t, t]
|
|
52
|
+
72 y [b, t, n_embd]
|
|
53
|
+
MLP.forward
|
|
54
|
+
88 x [b, t, 4 * n_embd]
|
|
55
|
+
GPT.forward
|
|
56
|
+
177 tok_emb [b, t, n_embd]
|
|
57
|
+
178 pos_emb [t, n_embd]
|
|
58
|
+
186 logits [b, t, vocab_size]
|
|
59
|
+
GPT.generate
|
|
60
|
+
321 v, _ [b, int]
|
|
61
|
+
326 idx_next [b, 1]
|
|
62
|
+
|
|
63
|
+
37 tensors, 32 fully named
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
```console
|
|
67
|
+
$ tensi check model.py
|
|
68
|
+
tensi check v0.3.0 model.py 2.0s
|
|
69
|
+
ok no shape errors
|
|
70
|
+
|
|
71
|
+
$ tensi check bad.py
|
|
72
|
+
tensi check v0.3.0 bad.py 819ms
|
|
73
|
+
x bad.py:17:9 Argument `Tensor[[b, 7, v]]` is not assignable to parameter `x` with type `Tensor[[b, 7, d]]` in function `Block.forward`
|
|
74
|
+
x bad.py:18:9 `@` is not supported between `Tensor[[b, d]]` and `Tensor[[v, 3]]`
|
|
75
|
+
|
|
76
|
+
2 shape errors
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
`tensi check src/` checks every Python file under a folder and exits 1 when it finds a shape error, so it
|
|
80
|
+
can gate CI.
|
|
81
|
+
|
|
82
|
+
## Contributing
|
|
83
|
+
|
|
84
|
+
See [CONTRIBUTING.md](https://github.com/tristan-farran/tensi/blob/main/CONTRIBUTING.md) for the development setup, tests and releases.
|
|
85
|
+
|
|
86
|
+
<sub>tensi is built on Pyrefly (MIT) and its torch stubs. The mascot is a small tensor that is very good at
|
|
87
|
+
looking.</sub>
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "tensi-shapes"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Named tensor shapes for unannotated PyTorch: hovers, shape errors and a CLI, built on Pyrefly."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.12"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Tristan Farran" }]
|
|
14
|
+
keywords = ["pytorch", "tensor", "shapes", "type-checking", "pyrefly", "jaxtyping", "einops", "lsp"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Intended Audience :: Science/Research",
|
|
20
|
+
"Operating System :: OS Independent",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
23
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
24
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
25
|
+
]
|
|
26
|
+
dependencies = [
|
|
27
|
+
"pyrefly==1.5.0",
|
|
28
|
+
"pyrefly-torch-stubs==1.5.0",
|
|
29
|
+
"pyrefly-einops-stubs==1.5.0",
|
|
30
|
+
"parso>=0.8.4",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.optional-dependencies]
|
|
34
|
+
test = ["pytest>=8", "jaxtyping", "einops"]
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
Homepage = "https://github.com/tristan-farran/tensi"
|
|
38
|
+
Issues = "https://github.com/tristan-farran/tensi/issues"
|
|
39
|
+
Changelog = "https://github.com/tristan-farran/tensi/blob/main/CHANGELOG.md"
|
|
40
|
+
|
|
41
|
+
[project.scripts]
|
|
42
|
+
tensi = "tensi.cli:main"
|
|
43
|
+
|
|
44
|
+
[tool.setuptools]
|
|
45
|
+
packages = ["tensi"]
|
|
46
|
+
|
|
47
|
+
[tool.setuptools.dynamic]
|
|
48
|
+
version = { attr = "tensi.__version__" }
|
|
49
|
+
|
|
50
|
+
[tool.pytest.ini_options]
|
|
51
|
+
testpaths = ["tests"]
|
|
52
|
+
|
|
53
|
+
[tool.ruff]
|
|
54
|
+
line-length = 120
|
|
55
|
+
target-version = "py310"
|
|
56
|
+
extend-exclude = ["tests/fixtures", "tests/golden"]
|
|
57
|
+
|
|
58
|
+
[tool.ruff.lint]
|
|
59
|
+
select = ["E9", "F", "I", "B", "UP"]
|
|
60
|
+
ignore = ["B905", "B023", "UP007"]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""tensi: named tensor shapes for PyTorch, on Pyrefly."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import shutil
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
__version__ = "0.3.0"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def pyrefly() -> str:
|
|
11
|
+
"""The Pyrefly executable: `$TENSI_PYREFLY`, else the one installed beside this Python (so a
|
|
12
|
+
venv's Pyrefly wins even when the venv is not activated), else `pyrefly` on PATH."""
|
|
13
|
+
env = os.environ.get("TENSI_PYREFLY")
|
|
14
|
+
if env:
|
|
15
|
+
return env
|
|
16
|
+
exe = "pyrefly.exe" if os.name == "nt" else "pyrefly"
|
|
17
|
+
beside = os.path.join(os.path.dirname(sys.executable), exe)
|
|
18
|
+
return beside if os.path.isfile(beside) else (shutil.which("pyrefly") or "pyrefly")
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
"""Command line: `tensi peek`, `tensi check`, `tensi shadow`, `tensi stubs`, `tensi doctor`, `tensi lsp`."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import subprocess
|
|
10
|
+
import sys
|
|
11
|
+
|
|
12
|
+
from . import __version__, pyrefly, stubs
|
|
13
|
+
from .mapping import PositionMap, clean_display
|
|
14
|
+
from .rewrite import rewrite
|
|
15
|
+
from .scratch import scratch_copy
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def is_shape_diagnostic(message: str, name: str | None = None) -> bool:
|
|
19
|
+
"""`shapes` diagnostics mode keeps only errors that are about tensor shapes.
|
|
20
|
+
|
|
21
|
+
A type mismatch counts only when the type it wanted is shaped too: a shaped tensor where code
|
|
22
|
+
annotates `LongTensor`, `int | ndarray` or `tuple[int]` is a typing complaint, not a shape error.
|
|
23
|
+
Nullability errors that merely mention a shaped type (`Tensor[[n]] | None` passed where
|
|
24
|
+
`Tensor[[n]]` is expected) and override checks are not about shapes either.
|
|
25
|
+
"""
|
|
26
|
+
if name == "bad-override" or re.search(r"\| None` is not assignable", message):
|
|
27
|
+
return False
|
|
28
|
+
if name == "bad-unpacking" or message.startswith("Cannot unpack"):
|
|
29
|
+
return "IntTuple[" in message # `b, t, c = x.shape` on a tensor of another rank
|
|
30
|
+
first = message.split("\n", 1)[0]
|
|
31
|
+
m = re.search(r"is not assignable to (?:variable|parameter|declared return type|attribute|type)?(.*)", first)
|
|
32
|
+
if m:
|
|
33
|
+
return "[[" in first[: m.start()] and "[[" in m.group(1)
|
|
34
|
+
m = re.search(r"No matching overload found for function .*? called with arguments: (.*)", first)
|
|
35
|
+
if m:
|
|
36
|
+
return "[[" in m.group(1)
|
|
37
|
+
return (
|
|
38
|
+
"Tensor[[" in message
|
|
39
|
+
or "shape DSL" in message
|
|
40
|
+
or ("shape" in message.lower() and "Tensor" in message)
|
|
41
|
+
or bool(re.search(r"\b(rank|dimensions?|broadcast)\b", first))
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def is_rewrite_artifact(message: str) -> bool:
|
|
46
|
+
"""Errors caused by the shadow itself, e.g. a config field `n_embd: Int[_ts_n_embd] = 768`."""
|
|
47
|
+
return bool(re.search(r"is not assignable to `(?:_ts_)?Int\[_ts_|Unnecessary `int\(\)` call", message))
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def is_imprecise(message: str) -> bool:
|
|
51
|
+
"""Errors that only exist because Pyrefly is unsure: a union of candidate types, Unknown/Any,
|
|
52
|
+
or unsolved type variables (`@_`), e.g. attribute access through an nn.ModuleDict.
|
|
53
|
+
Not confident enough to underline."""
|
|
54
|
+
return bool(re.search(r"\]\] \| |\| (Any|Unknown)\b|\bUnknown\b|@_", message)) or _divisible_only(message)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
_ROUNDED = re.compile(r"\b(\d+) \* \((\w+) // \1\)|\b(\w+) // (\d+) \* \4\b")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _divisible_only(message: str) -> bool:
|
|
61
|
+
"""`dim` against `4 * (dim // 4)`: equal whenever the code divides evenly (it usually asserts
|
|
62
|
+
`dim % 4 == 0`), which Pyrefly cannot know. Not an error worth underlining."""
|
|
63
|
+
for m in _ROUNDED.finditer(message):
|
|
64
|
+
name = m.group(2) or m.group(3)
|
|
65
|
+
if re.search(rf"(?<![\w/*] )(?<![\w(])\b{name}\b(?! //)", _ROUNDED.sub("", message)):
|
|
66
|
+
return True
|
|
67
|
+
return False
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def drop_imprecise(diags: list[dict], msg_key: str, range_key) -> list[dict]:
|
|
71
|
+
"""Drop imprecise diagnostics and everything else reported at the same range."""
|
|
72
|
+
bad_ranges = {range_key(d) for d in diags if is_imprecise(d[msg_key])}
|
|
73
|
+
return [d for d in diags if range_key(d) not in bad_ranges]
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def shadow_for(path: str, solve_seeds: bool = True):
|
|
77
|
+
"""(source, rewrite result) for `path`, with forward-input seeds solved unless disabled."""
|
|
78
|
+
from .seeds import solve
|
|
79
|
+
|
|
80
|
+
with open(path, encoding="utf-8") as f:
|
|
81
|
+
src = f.read()
|
|
82
|
+
seeds = solve(path, src) if solve_seeds else None
|
|
83
|
+
return src, rewrite(src, seeds=seeds)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def check_file(path: str, shapes_only: bool = True, preset: str = "default", solve_seeds: bool = True) -> list[dict]:
|
|
87
|
+
"""Run Pyrefly on the shadow of `path`; return diagnostics in user coordinates and names."""
|
|
88
|
+
src, res = shadow_for(path, solve_seeds)
|
|
89
|
+
pm = PositionMap(src, res.shadow, res.edits)
|
|
90
|
+
folder = os.path.dirname(os.path.abspath(path))
|
|
91
|
+
with scratch_copy(folder, res.shadow, "_tensi_tmp_") as (tmp, extra, cwd):
|
|
92
|
+
proc = subprocess.run(
|
|
93
|
+
[pyrefly(), "check", tmp, "--preset", preset, "--output-format", "json", *extra, *stubs.search_args()],
|
|
94
|
+
capture_output=True,
|
|
95
|
+
text=True,
|
|
96
|
+
cwd=cwd,
|
|
97
|
+
)
|
|
98
|
+
# Fail closed: a Pyrefly crash must not read as "no shape errors" in CI.
|
|
99
|
+
if proc.returncode not in (0, 1) or not proc.stdout.strip():
|
|
100
|
+
raise SystemExit(f"tensi: pyrefly failed (exit {proc.returncode}):\n{proc.stdout}{proc.stderr}")
|
|
101
|
+
try:
|
|
102
|
+
data = json.loads(proc.stdout)
|
|
103
|
+
except json.JSONDecodeError:
|
|
104
|
+
raise SystemExit(f"tensi: pyrefly failed:\n{proc.stdout}\n{proc.stderr}") from None
|
|
105
|
+
out = []
|
|
106
|
+
errors = data.get("errors", [])
|
|
107
|
+
errors = [e for e in errors if not is_rewrite_artifact(e.get("description", ""))]
|
|
108
|
+
if shapes_only:
|
|
109
|
+
errors = drop_imprecise(
|
|
110
|
+
errors, "description", lambda e: (e["line"], e["column"], e["stop_line"], e["stop_column"])
|
|
111
|
+
)
|
|
112
|
+
for e in errors:
|
|
113
|
+
msg = clean_display(e.get("concise_description") or e.get("description", ""))
|
|
114
|
+
if shapes_only and not is_shape_diagnostic(e.get("description", ""), e.get("name")):
|
|
115
|
+
continue
|
|
116
|
+
l0, l1 = e["line"] - 1, e["stop_line"] - 1
|
|
117
|
+
c0 = pm.shadow_to_user_cp(l0, e["column"] - 1)
|
|
118
|
+
c1 = pm.shadow_to_user_cp(l1, e["stop_column"] - 1)
|
|
119
|
+
out.append(
|
|
120
|
+
{
|
|
121
|
+
"path": path,
|
|
122
|
+
"line": l0 + 1,
|
|
123
|
+
"column": c0 + 1,
|
|
124
|
+
"stop_line": l1 + 1,
|
|
125
|
+
"stop_column": max(c1, c0) + 1,
|
|
126
|
+
"name": e.get("name"),
|
|
127
|
+
"message": msg,
|
|
128
|
+
}
|
|
129
|
+
)
|
|
130
|
+
return out
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _elapsed(t0: float) -> str:
|
|
134
|
+
import time
|
|
135
|
+
|
|
136
|
+
dt = time.perf_counter() - t0
|
|
137
|
+
return f"{dt * 1000:.0f}ms" if dt < 1 else f"{dt:.1f}s"
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def cmd_peek(path: str) -> int:
|
|
141
|
+
import time
|
|
142
|
+
|
|
143
|
+
from .peek import color_shape, peek, style_for
|
|
144
|
+
|
|
145
|
+
st = style_for(sys.stdout)
|
|
146
|
+
t0 = time.perf_counter()
|
|
147
|
+
rows, src = peek(path)
|
|
148
|
+
print(f"{st.plum('tensi')} {st.dim('peek v' + __version__)} {path} {st.dim(_elapsed(t0))}\n")
|
|
149
|
+
width = max((len(", ".join(r.names)) for r in rows), default=0)
|
|
150
|
+
width = min(width, 18)
|
|
151
|
+
last_fn = None
|
|
152
|
+
for r in rows:
|
|
153
|
+
fn = _enclosing_def(src, r.line)
|
|
154
|
+
if fn != last_fn:
|
|
155
|
+
print(f" {st.dim(fn or '<module>')}")
|
|
156
|
+
last_fn = fn
|
|
157
|
+
names = ", ".join(r.names)
|
|
158
|
+
print(f" {st.dim(f'{r.line:>4}')} {names:<{width}} {color_shape(r.shape, st)}")
|
|
159
|
+
named = sum(1 for r in rows if is_fully_named(r.shape))
|
|
160
|
+
print(f"\n {len(rows)} tensors, {st.violet(str(named))} fully named")
|
|
161
|
+
hints = getattr(rows, "hints", [])
|
|
162
|
+
if hints:
|
|
163
|
+
print(f"\n {st.dim('to name more, annotate the input of:')}")
|
|
164
|
+
for h in hints:
|
|
165
|
+
example = f'e.g. {h.param}: Shaped[Tensor, "{h.axes}"]'
|
|
166
|
+
known = f" (now {'[' + ', '.join(t.lstrip('~') for t in h.seed) + ']'})" if h.seed else ""
|
|
167
|
+
print(f" {st.dim(f'{h.line:>4}')} {h.cls}.forward({h.param}) {st.dim(example + known)}")
|
|
168
|
+
return 0
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def is_fully_named(shape: str) -> bool:
|
|
172
|
+
"""`[b, n_points]` yes; `[b, int]`, `[?]`, `[2, 3]` no. (A dim name may contain "int".)"""
|
|
173
|
+
return not re.search(r"\bint\b|\?", shape) and bool(re.search(r"[A-Za-z_]", shape))
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _enclosing_def(src: list[str], line: int) -> str | None:
|
|
177
|
+
"""`Class.method` / `func` of the innermost def above `line` (by indentation)."""
|
|
178
|
+
cur_indent = None
|
|
179
|
+
names = []
|
|
180
|
+
for i in range(line - 1, -1, -1):
|
|
181
|
+
text = src[i]
|
|
182
|
+
if not text.strip():
|
|
183
|
+
continue
|
|
184
|
+
indent = len(text) - len(text.lstrip())
|
|
185
|
+
m = re.match(r"\s*(?:async\s+)?(def|class)\s+(\w+)", text)
|
|
186
|
+
if m and (cur_indent is None or indent < cur_indent):
|
|
187
|
+
names.append(m.group(2))
|
|
188
|
+
cur_indent = indent
|
|
189
|
+
if indent == 0:
|
|
190
|
+
break
|
|
191
|
+
return ".".join(reversed(names)) if names else None
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def python_files(paths: list[str]) -> list[str]:
|
|
195
|
+
"""Files named on the command line, with folders expanded to the `.py` files under them
|
|
196
|
+
(skipping hidden folders, virtualenvs and caches)."""
|
|
197
|
+
skip = {"__pycache__", "node_modules", "site-packages", "build", "dist"}
|
|
198
|
+
out = []
|
|
199
|
+
for p in paths:
|
|
200
|
+
if os.path.isdir(p):
|
|
201
|
+
for root, dirs, files in os.walk(p):
|
|
202
|
+
dirs[:] = sorted(
|
|
203
|
+
d
|
|
204
|
+
for d in dirs
|
|
205
|
+
if not d.startswith(".")
|
|
206
|
+
and d not in skip
|
|
207
|
+
and not os.path.exists(os.path.join(root, d, "pyvenv.cfg"))
|
|
208
|
+
)
|
|
209
|
+
out += [os.path.join(root, f) for f in sorted(files) if f.endswith(".py")]
|
|
210
|
+
elif os.path.isfile(p):
|
|
211
|
+
out.append(p)
|
|
212
|
+
else:
|
|
213
|
+
raise SystemExit(f"tensi: no such file or folder: {p}")
|
|
214
|
+
return out
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def cmd_check(paths: list[str], show_all: bool, solve_seeds: bool) -> int:
|
|
218
|
+
import time
|
|
219
|
+
|
|
220
|
+
from .peek import style_for
|
|
221
|
+
|
|
222
|
+
st = style_for(sys.stdout)
|
|
223
|
+
files = python_files(paths)
|
|
224
|
+
total = 0
|
|
225
|
+
for path in files:
|
|
226
|
+
t0 = time.perf_counter()
|
|
227
|
+
diags = check_file(path, shapes_only=not show_all, solve_seeds=solve_seeds)
|
|
228
|
+
print(f"{st.plum('tensi')} {st.dim('check v' + __version__)} {path} {st.dim(_elapsed(t0))}")
|
|
229
|
+
for d in diags:
|
|
230
|
+
loc = f"{d['path']}:{d['line']}:{d['column']}"
|
|
231
|
+
print(f" {st.red('x')} {st.bold(loc)} {d['message']} {st.dim(d['name'] or '')}")
|
|
232
|
+
total += len(diags)
|
|
233
|
+
kind = "" if show_all else "shape "
|
|
234
|
+
where = f" in {len(files)} files" if len(files) > 1 else ""
|
|
235
|
+
if total:
|
|
236
|
+
print(f"\n {st.red(str(total))} {kind}error{'s' if total != 1 else ''}{where}")
|
|
237
|
+
return 1
|
|
238
|
+
print(f" {st.green('ok')} no {kind}errors{where}")
|
|
239
|
+
return 0
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def main(argv: list[str] | None = None) -> int:
|
|
243
|
+
ap = argparse.ArgumentParser(
|
|
244
|
+
prog="tensi", description="tensi: see your tensors. Named shapes for PyTorch, on Pyrefly."
|
|
245
|
+
)
|
|
246
|
+
ap.add_argument("--version", action="version", version=f"tensi {__version__}")
|
|
247
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
248
|
+
pk = sub.add_parser("peek", help="print the shape of every tensor in a file")
|
|
249
|
+
pk.add_argument("file")
|
|
250
|
+
ck = sub.add_parser("check", help="report tensor-shape errors in files or folders (exit 1 if any)")
|
|
251
|
+
ck.add_argument("paths", nargs="+", metavar="path")
|
|
252
|
+
ck.add_argument("--all", action="store_true", help="all type errors, not only shape errors")
|
|
253
|
+
ck.add_argument("--no-solve", action="store_true", help="skip the forward-input seed solver")
|
|
254
|
+
sh = sub.add_parser("shadow", help="print the rewritten text Pyrefly sees (debugging)")
|
|
255
|
+
sh.add_argument("file")
|
|
256
|
+
sh.add_argument("--notes", action="store_true", help="print rewriter notes to stderr")
|
|
257
|
+
sh.add_argument("--no-solve", action="store_true", help="skip the forward-input seed solver")
|
|
258
|
+
stb = sub.add_parser("stubs", help="show tensi's torch stub patches (--diff: as a patch for upstream)")
|
|
259
|
+
stb.add_argument("--diff", action="store_true")
|
|
260
|
+
dr = sub.add_parser("doctor", help="check the setup (Pyrefly, stubs, torch) from a folder")
|
|
261
|
+
dr.add_argument("folder", nargs="?", default=".")
|
|
262
|
+
lsp = sub.add_parser("lsp", help="run the language server on stdio")
|
|
263
|
+
lsp.add_argument("--pyrefly", default=pyrefly())
|
|
264
|
+
args = ap.parse_args(argv)
|
|
265
|
+
|
|
266
|
+
if args.cmd == "peek":
|
|
267
|
+
python_files([args.file]) # a clean message for a missing file
|
|
268
|
+
return cmd_peek(args.file)
|
|
269
|
+
if args.cmd == "check":
|
|
270
|
+
return cmd_check(args.paths, args.all, not args.no_solve)
|
|
271
|
+
if args.cmd == "shadow":
|
|
272
|
+
_src, res = shadow_for(args.file, not args.no_solve)
|
|
273
|
+
sys.stdout.write(res.shadow)
|
|
274
|
+
if args.notes:
|
|
275
|
+
for n in res.notes:
|
|
276
|
+
print(f"note: {n}", file=sys.stderr)
|
|
277
|
+
return 0
|
|
278
|
+
if args.cmd == "stubs":
|
|
279
|
+
if args.diff:
|
|
280
|
+
sys.stdout.write(stubs.diff())
|
|
281
|
+
return 0
|
|
282
|
+
print(stubs.status(), end="")
|
|
283
|
+
for p in stubs.PATCHES:
|
|
284
|
+
print(f" {p.name:<20} {p.why}")
|
|
285
|
+
return 0
|
|
286
|
+
if args.cmd == "doctor":
|
|
287
|
+
from .doctor import run
|
|
288
|
+
|
|
289
|
+
return run(args.folder)
|
|
290
|
+
if args.cmd == "lsp":
|
|
291
|
+
from .server import serve
|
|
292
|
+
|
|
293
|
+
return serve(args.pyrefly)
|
|
294
|
+
return 2
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
if __name__ == "__main__":
|
|
298
|
+
raise SystemExit(main())
|