lint-tool 2026.9.14__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. lint_tool-2026.9.14/LICENSE +15 -0
  2. lint_tool-2026.9.14/PKG-INFO +76 -0
  3. lint_tool-2026.9.14/README.md +49 -0
  4. lint_tool-2026.9.14/pyproject.toml +49 -0
  5. lint_tool-2026.9.14/setup.cfg +4 -0
  6. lint_tool-2026.9.14/src/lint_tool/__init__.py +3 -0
  7. lint_tool-2026.9.14/src/lint_tool/__main__.py +16 -0
  8. lint_tool-2026.9.14/src/lint_tool/adapters/__init__.py +0 -0
  9. lint_tool-2026.9.14/src/lint_tool/adapters/_linter/__init__.py +45 -0
  10. lint_tool-2026.9.14/src/lint_tool/adapters/_linter/argument_parser.py +47 -0
  11. lint_tool-2026.9.14/src/lint_tool/adapters/_linter/block.py +179 -0
  12. lint_tool-2026.9.14/src/lint_tool/adapters/_linter/blocks.py +129 -0
  13. lint_tool-2026.9.14/src/lint_tool/adapters/_linter/bracket_pairs.py +45 -0
  14. lint_tool-2026.9.14/src/lint_tool/adapters/_linter/file_linter.py +191 -0
  15. lint_tool-2026.9.14/src/lint_tool/adapters/_linter/host.py +124 -0
  16. lint_tool-2026.9.14/src/lint_tool/adapters/_linter/messages.py +106 -0
  17. lint_tool-2026.9.14/src/lint_tool/adapters/_linter/python_file.py +190 -0
  18. lint_tool-2026.9.14/src/lint_tool/adapters/_linter/sets.py +73 -0
  19. lint_tool-2026.9.14/src/lint_tool/adapters/actionlint_linter.py +166 -0
  20. lint_tool-2026.9.14/src/lint_tool/adapters/bazel_linter.py +194 -0
  21. lint_tool-2026.9.14/src/lint_tool/adapters/clangformat_linter.py +245 -0
  22. lint_tool-2026.9.14/src/lint_tool/adapters/clangtidy_linter.py +435 -0
  23. lint_tool-2026.9.14/src/lint_tool/adapters/cmake_format_linter.py +286 -0
  24. lint_tool-2026.9.14/src/lint_tool/adapters/cmake_linter.py +140 -0
  25. lint_tool-2026.9.14/src/lint_tool/adapters/cmake_minimum_required_linter.py +249 -0
  26. lint_tool-2026.9.14/src/lint_tool/adapters/codespell_linter.py +255 -0
  27. lint_tool-2026.9.14/src/lint_tool/adapters/docstring_linter-grandfather.json +267 -0
  28. lint_tool-2026.9.14/src/lint_tool/adapters/docstring_linter.py +295 -0
  29. lint_tool-2026.9.14/src/lint_tool/adapters/editorconfig_checker_linter.py +218 -0
  30. lint_tool-2026.9.14/src/lint_tool/adapters/exec_linter.py +89 -0
  31. lint_tool-2026.9.14/src/lint_tool/adapters/flake8_linter.py +367 -0
  32. lint_tool-2026.9.14/src/lint_tool/adapters/gha_linter.py +95 -0
  33. lint_tool-2026.9.14/src/lint_tool/adapters/grep_linter.py +283 -0
  34. lint_tool-2026.9.14/src/lint_tool/adapters/import_linter.py +145 -0
  35. lint_tool-2026.9.14/src/lint_tool/adapters/lintrunner_version_linter.py +82 -0
  36. lint_tool-2026.9.14/src/lint_tool/adapters/mypy_linter.py +292 -0
  37. lint_tool-2026.9.14/src/lint_tool/adapters/newlines_linter.py +166 -0
  38. lint_tool-2026.9.14/src/lint_tool/adapters/pip_init.py +105 -0
  39. lint_tool-2026.9.14/src/lint_tool/adapters/pyfmt_linter.py +178 -0
  40. lint_tool-2026.9.14/src/lint_tool/adapters/pyproject_linter.py +249 -0
  41. lint_tool-2026.9.14/src/lint_tool/adapters/pyrefly_linter.py +257 -0
  42. lint_tool-2026.9.14/src/lint_tool/adapters/ruff_linter.py +462 -0
  43. lint_tool-2026.9.14/src/lint_tool/adapters/s3_init.py +225 -0
  44. lint_tool-2026.9.14/src/lint_tool/adapters/s3_init_config.json +61 -0
  45. lint_tool-2026.9.14/src/lint_tool/adapters/shellcheck_linter.py +120 -0
  46. lint_tool-2026.9.14/src/lint_tool/adapters/update_s3.py +98 -0
  47. lint_tool-2026.9.14/src/lint_tool.egg-info/PKG-INFO +76 -0
  48. lint_tool-2026.9.14/src/lint_tool.egg-info/SOURCES.txt +51 -0
  49. lint_tool-2026.9.14/src/lint_tool.egg-info/dependency_links.txt +1 -0
  50. lint_tool-2026.9.14/src/lint_tool.egg-info/entry_points.txt +2 -0
  51. lint_tool-2026.9.14/src/lint_tool.egg-info/requires.txt +6 -0
  52. lint_tool-2026.9.14/src/lint_tool.egg-info/top_level.txt +1 -0
  53. lint_tool-2026.9.14/tests/test_host.py +69 -0
@@ -0,0 +1,15 @@
1
+ Copyright (c) 2025-2026 KhwarizmiAnalytix
2
+
3
+ This program is free software: you can redistribute it and/or modify
4
+ it under the terms of the GNU General Public License as published by
5
+ the Free Software Foundation, either version 3 of the License, or
6
+ (at your option) any later version.
7
+
8
+ This program is distributed in the hope that it will be useful,
9
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
10
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
+ GNU General Public License for more details.
12
+
13
+ SPDX-License-Identifier: GPL-3.0-or-later
14
+
15
+ Full license text: https://www.gnu.org/licenses/gpl-3.0.html
@@ -0,0 +1,76 @@
1
+ Metadata-Version: 2.4
2
+ Name: lint-tool
3
+ Version: 2026.9.14
4
+ Summary: lintrunner adapters for C++/Python/CMake repositories
5
+ Author: KhwarizmiAnalytix
6
+ License-Expression: GPL-3.0-or-later
7
+ Project-URL: Homepage, https://github.com/KhwarizmiAnalytix/lint-tool
8
+ Project-URL: Source, https://github.com/KhwarizmiAnalytix/lint-tool
9
+ Project-URL: Issues, https://github.com/KhwarizmiAnalytix/lint-tool/issues
10
+ Keywords: lintrunner,lint,clang-format,ruff,cmake
11
+ Classifier: Development Status :: 5 - Production/Stable
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.9
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Topic :: Software Development :: Quality Assurance
20
+ Requires-Python: >=3.9
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Requires-Dist: tomli>=2.0; python_version < "3.11"
24
+ Provides-Extra: test
25
+ Requires-Dist: pytest>=7.0; extra == "test"
26
+ Dynamic: license-file
27
+
28
+ # lint-tool
29
+
30
+ lintrunner adapters for C++/Python/CMake repositories. Host repos own
31
+ policy (`.lintrunner.toml`, `.clang-format`, `pyproject.toml`); this
32
+ package owns the adapter implementations.
33
+
34
+ ```bash
35
+ pip install lint-tool
36
+ ```
37
+
38
+ Invoke adapters through lintrunner:
39
+
40
+ ```toml
41
+ [[linter]]
42
+ code = 'CLANGFORMAT'
43
+ include_patterns = ['**/*.cpp', '**/*.h']
44
+ exclude_patterns = ['ThirdParty/**']
45
+ command = [
46
+ 'python3',
47
+ '-m',
48
+ 'lint_tool.adapters.clangformat_linter',
49
+ '--binary=clang-format',
50
+ '--',
51
+ '@{{PATHSFILE}}',
52
+ ]
53
+ init_command = [
54
+ 'python3',
55
+ '-m',
56
+ 'lint_tool.adapters.pip_init',
57
+ '--dry-run={{DRYRUN}}',
58
+ 'clang-format==19.1.4',
59
+ ]
60
+ ```
61
+
62
+ Adapters find the consumer root by walking up from cwd for
63
+ `.lintrunner.toml` or `.git`, or from `LINTRUNNER_HOST_ROOT`. They do not
64
+ assume an XSigma `Tools/linter/adapters` layout.
65
+
66
+ A Logging/Parallel-sized menu is typically CLANGFORMAT, CMAKE,
67
+ CMAKEFORMAT, EDITORCONFIG, NEWLINE, and CODESPELL. Do not copy a full
68
+ XSigma `.lintrunner.toml`.
69
+
70
+ ```bash
71
+ python3 -m lint_tool # list usage
72
+ python3 -m lint_tool.adapters.ruff_linter --help
73
+ python3 -m pytest tests/ # from a clone, after pip install -e ".[test]"
74
+ ```
75
+
76
+ Source: https://github.com/KhwarizmiAnalytix/lint-tool
@@ -0,0 +1,49 @@
1
+ # lint-tool
2
+
3
+ lintrunner adapters for C++/Python/CMake repositories. Host repos own
4
+ policy (`.lintrunner.toml`, `.clang-format`, `pyproject.toml`); this
5
+ package owns the adapter implementations.
6
+
7
+ ```bash
8
+ pip install lint-tool
9
+ ```
10
+
11
+ Invoke adapters through lintrunner:
12
+
13
+ ```toml
14
+ [[linter]]
15
+ code = 'CLANGFORMAT'
16
+ include_patterns = ['**/*.cpp', '**/*.h']
17
+ exclude_patterns = ['ThirdParty/**']
18
+ command = [
19
+ 'python3',
20
+ '-m',
21
+ 'lint_tool.adapters.clangformat_linter',
22
+ '--binary=clang-format',
23
+ '--',
24
+ '@{{PATHSFILE}}',
25
+ ]
26
+ init_command = [
27
+ 'python3',
28
+ '-m',
29
+ 'lint_tool.adapters.pip_init',
30
+ '--dry-run={{DRYRUN}}',
31
+ 'clang-format==19.1.4',
32
+ ]
33
+ ```
34
+
35
+ Adapters find the consumer root by walking up from cwd for
36
+ `.lintrunner.toml` or `.git`, or from `LINTRUNNER_HOST_ROOT`. They do not
37
+ assume an XSigma `Tools/linter/adapters` layout.
38
+
39
+ A Logging/Parallel-sized menu is typically CLANGFORMAT, CMAKE,
40
+ CMAKEFORMAT, EDITORCONFIG, NEWLINE, and CODESPELL. Do not copy a full
41
+ XSigma `.lintrunner.toml`.
42
+
43
+ ```bash
44
+ python3 -m lint_tool # list usage
45
+ python3 -m lint_tool.adapters.ruff_linter --help
46
+ python3 -m pytest tests/ # from a clone, after pip install -e ".[test]"
47
+ ```
48
+
49
+ Source: https://github.com/KhwarizmiAnalytix/lint-tool
@@ -0,0 +1,49 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "lint-tool"
7
+ version = "2026.9.14"
8
+ description = "lintrunner adapters for C++/Python/CMake repositories"
9
+ readme = "README.md"
10
+ requires-python = ">=3.9"
11
+ license = "GPL-3.0-or-later"
12
+ authors = [
13
+ { name = "KhwarizmiAnalytix" },
14
+ ]
15
+ keywords = ["lintrunner", "lint", "clang-format", "ruff", "cmake"]
16
+ classifiers = [
17
+ "Development Status :: 5 - Production/Stable",
18
+ "Environment :: Console",
19
+ "Intended Audience :: Developers",
20
+ "Programming Language :: Python :: 3",
21
+ "Programming Language :: Python :: 3.9",
22
+ "Programming Language :: Python :: 3.10",
23
+ "Programming Language :: Python :: 3.11",
24
+ "Programming Language :: Python :: 3.12",
25
+ "Topic :: Software Development :: Quality Assurance",
26
+ ]
27
+ dependencies = [
28
+ "tomli>=2.0; python_version < '3.11'",
29
+ ]
30
+
31
+ [project.optional-dependencies]
32
+ test = ["pytest>=7.0"]
33
+
34
+ [project.urls]
35
+ Homepage = "https://github.com/KhwarizmiAnalytix/lint-tool"
36
+ Source = "https://github.com/KhwarizmiAnalytix/lint-tool"
37
+ Issues = "https://github.com/KhwarizmiAnalytix/lint-tool/issues"
38
+
39
+ [project.scripts]
40
+ lint-tool = "lint_tool.__main__:main"
41
+
42
+ [tool.setuptools.packages.find]
43
+ where = ["src"]
44
+
45
+ [tool.setuptools.package-data]
46
+ "lint_tool.adapters" = ["*.json"]
47
+
48
+ [tool.pytest.ini_options]
49
+ testpaths = ["tests"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,3 @@
1
+ """lintrunner adapter suite for KhwarizmiAnalytix repositories."""
2
+
3
+ __version__ = "2026.9.14"
@@ -0,0 +1,16 @@
1
+ """``python -m lint_tool`` prints how to invoke adapters."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from lint_tool import __version__
6
+
7
+
8
+ def main() -> None:
9
+ print(f"lint-tool {__version__}")
10
+ print("Run an adapter with:")
11
+ print(" python3 -m lint_tool.adapters.<name> --help")
12
+ print("Examples: clangformat_linter, ruff_linter, pip_init, s3_init")
13
+
14
+
15
+ if __name__ == "__main__":
16
+ main()
File without changes
@@ -0,0 +1,45 @@
1
+ from __future__ import annotations
2
+
3
+ import token
4
+ from typing import TYPE_CHECKING
5
+
6
+
7
+ if TYPE_CHECKING:
8
+ from collections.abc import Sequence
9
+ from tokenize import TokenInfo
10
+
11
+
12
+ __all__ = (
13
+ "Block",
14
+ "EMPTY_TOKENS",
15
+ "FileLinter",
16
+ "LineWithSets",
17
+ "LintResult",
18
+ "ParseError",
19
+ "PythonFile",
20
+ "ROOT",
21
+ "host_root",
22
+ )
23
+
24
+ NO_TOKEN = -1
25
+
26
+ # Python 3.12 and up have two new token types, FSTRING_START and FSTRING_END
27
+ _START_OF_LINE_TOKENS = token.DEDENT, token.INDENT, token.NEWLINE
28
+ _IGNORED_TOKENS = token.COMMENT, token.ENDMARKER, token.ENCODING, token.NL
29
+ EMPTY_TOKENS = dict.fromkeys(_START_OF_LINE_TOKENS + _IGNORED_TOKENS)
30
+
31
+ from .host import host_root
32
+
33
+ ROOT = host_root()
34
+
35
+
36
+ class ParseError(ValueError):
37
+ def __init__(self, token: TokenInfo, *args: str) -> None:
38
+ super().__init__(*args)
39
+ self.token = token
40
+
41
+
42
+ from .block import Block
43
+ from .file_linter import FileLinter
44
+ from .messages import LintResult
45
+ from .python_file import PythonFile
@@ -0,0 +1,47 @@
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import sys
5
+ from typing import Any
6
+ from typing_extensions import Never
7
+
8
+
9
+ class ArgumentParser(argparse.ArgumentParser):
10
+ """
11
+ Adds better help formatting and default arguments to argparse.ArgumentParser
12
+ """
13
+
14
+ def __init__(
15
+ self,
16
+ prog: str | None = None,
17
+ usage: str | None = None,
18
+ description: str | None = None,
19
+ epilog: str | None = None,
20
+ is_fixer: bool = False,
21
+ **kwargs: Any,
22
+ ) -> None:
23
+ super().__init__(prog, usage, description, None, **kwargs)
24
+ self._epilog = epilog
25
+
26
+ help = "A list of files or directories to lint"
27
+ self.add_argument("files", nargs="*", help=help)
28
+ # TODO(rec): get fromfile_prefix_chars="@", type=argparse.FileType to work
29
+
30
+ help = "Fix lint errors if possible" if is_fixer else argparse.SUPPRESS
31
+ self.add_argument("-f", "--fix", action="store_true", help=help)
32
+
33
+ help = "Run for lintrunner and print LintMessages which aren't edits"
34
+ self.add_argument("-l", "--lintrunner", action="store_true", help=help)
35
+
36
+ help = "Print more debug info"
37
+ self.add_argument("-v", "--verbose", action="store_true", help=help)
38
+
39
+ def exit(self, status: int = 0, message: str | None = None) -> Never:
40
+ """
41
+ Overriding this method is a workaround for argparse throwing away all
42
+ line breaks when printing the `epilog` section of the help message.
43
+ """
44
+ argv = sys.argv[1:]
45
+ if self._epilog and not status and "-h" in argv or "--help" in argv:
46
+ print(self._epilog)
47
+ super().exit(status, message)
@@ -0,0 +1,179 @@
1
+ from __future__ import annotations
2
+
3
+ import dataclasses as dc
4
+ import itertools
5
+ import token
6
+ from enum import Enum
7
+ from functools import cached_property, total_ordering
8
+ from typing import TYPE_CHECKING, Any, Optional
9
+ from typing_extensions import Self
10
+
11
+
12
+ if TYPE_CHECKING:
13
+ from collections.abc import Iterator, Sequence
14
+ from tokenize import TokenInfo
15
+
16
+
17
+ _OVERRIDES = {"@override", "@typing_extensions.override", "@typing.override"}
18
+
19
+
20
+ @total_ordering
21
+ @dc.dataclass
22
+ class Block:
23
+ """A block of Python code starting with either `def` or `class`"""
24
+
25
+ class Category(str, Enum):
26
+ CLASS = "class"
27
+ DEF = "def"
28
+
29
+ category: Category
30
+
31
+ # The sequence of tokens that contains this Block.
32
+ # Tokens are represented in `Block` as indexes into `self.tokens`
33
+ tokens: Sequence[TokenInfo] = dc.field(repr=False)
34
+
35
+ # The name of the function or class being defined
36
+ name: str
37
+
38
+ # The index of the very first token in the block (the "class" or "def" keyword)
39
+ begin: int
40
+
41
+ # The index of the first INDENT token for this block
42
+ indent: int
43
+
44
+ # The index of the DEDENT token for this end of this block
45
+ dedent: int
46
+
47
+ # The docstring for the block
48
+ docstring: str
49
+
50
+ # These next members only get filled in after all blocks have been constructed
51
+ # and figure out family ties
52
+
53
+ # The full qualified name of the block within the file.
54
+ # This is the name of this block and all its parents, joined with `.`.
55
+ full_name: str = ""
56
+
57
+ # The index of this block within the full list of blocks in the file
58
+ index: int = 0
59
+
60
+ # Is this block contained within a function definition?
61
+ is_local: bool = dc.field(default=False, repr=False)
62
+
63
+ # Is this block a function definition in a class definition?
64
+ is_method: bool = dc.field(default=False, repr=False)
65
+
66
+ # A block index to the parent of this block, or None for a top-level block.
67
+ parent: Optional[int] = None
68
+
69
+ # A list of block indexes for the children
70
+ children: list[int] = dc.field(default_factory=list)
71
+
72
+ @property
73
+ def start_line(self) -> int:
74
+ """The line number for the def or class statement"""
75
+ return self.tokens[self.begin].start[0]
76
+
77
+ @property
78
+ def end_line(self) -> int:
79
+ if 0 <= self.dedent < len(self.tokens):
80
+ return self.tokens[self.dedent].start[0] - 1
81
+ else:
82
+ return self.tokens[-1].start[0]
83
+ # Only happens in one case so far: a file whose last line was
84
+ #
85
+ # def function(): ...
86
+ #
87
+ # and the dedent correctly pointed to one past the end of self.tokens
88
+
89
+ @property
90
+ def line_count(self) -> int:
91
+ return self.end_line - self.start_line
92
+
93
+ @property
94
+ def is_class(self) -> bool:
95
+ return self.category == Block.Category.CLASS
96
+
97
+ @property
98
+ def display_name(self) -> str:
99
+ """A user-friendly name like 'class One' or 'def One.method()'"""
100
+ ending = "" if self.is_class else "()"
101
+ return f"{self.category.value} {self.full_name}{ending}"
102
+
103
+ @cached_property
104
+ def decorators(self) -> list[str]:
105
+ """A list of decorators for this function or method.
106
+
107
+ Each decorator both the @ symbol and any arguments to the decorator
108
+ but no extra whitespace.
109
+ """
110
+ return _get_decorators(self.tokens, self.begin)
111
+
112
+ @cached_property
113
+ def is_override(self) -> bool:
114
+ return not self.is_class and bool(_OVERRIDES.intersection(self.decorators))
115
+
116
+ DATA_FIELDS = (
117
+ "category",
118
+ "children",
119
+ "decorators",
120
+ "display_name",
121
+ "docstring",
122
+ "full_name",
123
+ "index",
124
+ "is_local",
125
+ "is_method",
126
+ "line_count",
127
+ "parent",
128
+ "start_line",
129
+ )
130
+
131
+ def as_data(self) -> dict[str, Any]:
132
+ d = {i: getattr(self, i) for i in self.DATA_FIELDS}
133
+ d["category"] = d["category"].value
134
+ return d
135
+
136
+ @property
137
+ def is_init(self) -> bool:
138
+ return not self.is_class and self.name == "__init__"
139
+
140
+ def contains(self, b: Block) -> bool:
141
+ return self.start_line < b.start_line and self.end_line >= b.end_line
142
+
143
+ def __eq__(self, o: object) -> bool:
144
+ assert isinstance(o, Block)
145
+ return o.tokens is self.tokens and o.index == self.index
146
+
147
+ def __hash__(self) -> int:
148
+ return super().__hash__()
149
+
150
+ def __lt__(self, o: Self) -> bool:
151
+ assert isinstance(o, Block) and o.tokens is self.tokens
152
+ return o.index < self.index
153
+
154
+
155
+ _IGNORE = {token.COMMENT, token.DEDENT, token.INDENT, token.NL}
156
+
157
+
158
+ def _get_decorators(tokens: Sequence[TokenInfo], block_start: int) -> list[str]:
159
+ def decorators() -> Iterator[str]:
160
+ rev = reversed(range(block_start))
161
+ newlines = (i for i in rev if tokens[i].type == token.NEWLINE)
162
+ it = iter(itertools.chain(newlines, [-1]))
163
+ # The -1 accounts for the very first line in the file
164
+
165
+ end = next(it, -1) # Like itertools.pairwise in Python 3.10
166
+ for begin in it:
167
+ for i in range(begin + 1, end):
168
+ t = tokens[i]
169
+ if t.type == token.OP and t.string == "@":
170
+ useful = (t for t in tokens[i:end] if t.type not in _IGNORE)
171
+ yield "".join(s.string.strip("\n") for s in useful)
172
+ break
173
+ elif t.type not in _IGNORE:
174
+ return # A statement means no more decorators
175
+ end = begin
176
+
177
+ out = list(decorators())
178
+ out.reverse()
179
+ return out
@@ -0,0 +1,129 @@
1
+ from __future__ import annotations
2
+
3
+ import token
4
+ from typing import TYPE_CHECKING, NamedTuple
5
+
6
+ from . import EMPTY_TOKENS, ParseError
7
+ from .block import Block
8
+
9
+
10
+ if TYPE_CHECKING:
11
+ from collections.abc import Sequence
12
+ from tokenize import TokenInfo
13
+
14
+
15
+ class BlocksResult(NamedTuple):
16
+ blocks: list[Block]
17
+ errors: dict[str, str]
18
+
19
+
20
+ def blocks(tokens: Sequence[TokenInfo]) -> BlocksResult:
21
+ blocks: list[Block] = []
22
+ indent_to_dedent = _make_indent_dict(tokens)
23
+ errors: dict[str, str] = {}
24
+
25
+ def starts_block(t: TokenInfo) -> bool:
26
+ return t.type == token.NAME and t.string in ("class", "def")
27
+
28
+ it = (i for i, t in enumerate(tokens) if starts_block(t))
29
+ blocks = [_make_block(tokens, i, indent_to_dedent, errors) for i in it]
30
+
31
+ for i, parent in enumerate(blocks):
32
+ for j in range(i + 1, len(blocks)):
33
+ if parent.contains(child := blocks[j]):
34
+ child.parent = i
35
+ parent.children.append(j)
36
+ else:
37
+ break
38
+
39
+ for i, b in enumerate(blocks):
40
+ b.index = i
41
+ parents = [b]
42
+ while (p := parents[-1].parent) is not None:
43
+ parents.append(blocks[p])
44
+ parents = parents[1:]
45
+
46
+ b.is_local = not all(p.is_class for p in parents)
47
+ b.is_method = not b.is_class and bool(parents) and parents[0].is_class
48
+
49
+ _add_full_names(blocks, [b for b in blocks if b.parent is None])
50
+ return BlocksResult(blocks, errors)
51
+
52
+
53
+ def _make_indent_dict(tokens: Sequence[TokenInfo]) -> dict[int, int]:
54
+ dedents = dict[int, int]()
55
+ stack = list[int]()
56
+
57
+ for i, t in enumerate(tokens):
58
+ if t.type == token.INDENT:
59
+ stack.append(i)
60
+ elif t.type == token.DEDENT:
61
+ dedents[stack.pop()] = i
62
+
63
+ return dedents
64
+
65
+
66
+ def _docstring(tokens: Sequence[TokenInfo], start: int) -> str:
67
+ for i in range(start + 1, len(tokens)):
68
+ tk = tokens[i]
69
+ if tk.type == token.STRING:
70
+ return tk.string
71
+ if tk.type not in EMPTY_TOKENS:
72
+ return ""
73
+ return ""
74
+
75
+
76
+ def _add_full_names(
77
+ blocks: Sequence[Block], children: Sequence[Block], prefix: str = ""
78
+ ) -> None:
79
+ # Would be trivial except that there can be duplicate names at any level
80
+ dupes: dict[str, list[Block]] = {}
81
+ for b in children:
82
+ dupes.setdefault(b.name, []).append(b)
83
+
84
+ for dl in dupes.values():
85
+ for i, b in enumerate(dl):
86
+ suffix = f"[{i + 1}]" if len(dl) > 1 else ""
87
+ b.full_name = prefix + b.name + suffix
88
+
89
+ for b in children:
90
+ if kids := [blocks[i] for i in b.children]:
91
+ _add_full_names(blocks, kids, b.full_name + ".")
92
+
93
+
94
+ def _make_block(
95
+ tokens: Sequence[TokenInfo],
96
+ begin: int,
97
+ indent_to_dedent: dict[int, int],
98
+ errors: dict[str, str],
99
+ ) -> Block:
100
+ def next_token(start: int, token_type: int, error: str) -> int:
101
+ for i in range(start, len(tokens)):
102
+ if tokens[i].type == token_type:
103
+ return i
104
+ raise ParseError(tokens[-1], error)
105
+
106
+ t = tokens[begin]
107
+ category = Block.Category[t.string.upper()]
108
+ indent = -1
109
+ dedent = -1
110
+ docstring = ""
111
+ name = "(not found)"
112
+ try:
113
+ ni = next_token(begin + 1, token.NAME, "Definition but no name")
114
+ name = tokens[ni].string
115
+ indent = next_token(ni + 1, token.INDENT, "Definition but no indent")
116
+ dedent = indent_to_dedent[indent]
117
+ docstring = _docstring(tokens, indent)
118
+ except ParseError as e:
119
+ errors[t.line] = " ".join(e.args)
120
+
121
+ return Block(
122
+ begin=begin,
123
+ category=category,
124
+ dedent=dedent,
125
+ docstring=docstring,
126
+ indent=indent,
127
+ name=name,
128
+ tokens=tokens,
129
+ )
@@ -0,0 +1,45 @@
1
+ import token
2
+ from collections.abc import Sequence
3
+ from tokenize import TokenInfo
4
+
5
+ from . import NO_TOKEN, ParseError
6
+
7
+
8
+ FSTRING_START: int = getattr(token, "FSTRING_START", NO_TOKEN)
9
+ FSTRING_END: int = getattr(token, "FSTRING_END", NO_TOKEN)
10
+
11
+ BRACKETS = {"{": "}", "(": ")", "[": "]"}
12
+ BRACKETS_INV = {j: i for i, j in BRACKETS.items()}
13
+
14
+
15
+ def bracket_pairs(tokens: Sequence[TokenInfo]) -> dict[int, int]:
16
+ """Returns a dictionary mapping opening to closing brackets"""
17
+ braces: dict[int, int] = {}
18
+ stack: list[int] = []
19
+ in_fstring = False
20
+
21
+ for i, t in enumerate(tokens):
22
+ if t.type == token.OP and not in_fstring:
23
+ if t.string in BRACKETS:
24
+ stack.append(i)
25
+ elif inv := BRACKETS_INV.get(t.string):
26
+ if not stack:
27
+ raise ParseError(t, "Never opened")
28
+ begin = stack.pop()
29
+
30
+ if not (stack and stack[-1] == FSTRING_START):
31
+ braces[begin] = i
32
+
33
+ b = tokens[begin].string
34
+ if b != inv:
35
+ raise ParseError(t, f"Mismatched braces '{b}' at {begin}")
36
+ elif t.type == FSTRING_START:
37
+ stack.append(FSTRING_START)
38
+ in_fstring = True
39
+ elif t.type == FSTRING_END:
40
+ if stack.pop() != FSTRING_START:
41
+ raise ParseError(t, "Mismatched FSTRING_START/FSTRING_END")
42
+ in_fstring = False
43
+ if stack:
44
+ raise ParseError(t, "Left open")
45
+ return braces