underscope 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- underscope/__init__.py +5 -0
- underscope/__main__.py +6 -0
- underscope/cli.py +2016 -0
- underscope-0.1.0.dist-info/METADATA +75 -0
- underscope-0.1.0.dist-info/RECORD +7 -0
- underscope-0.1.0.dist-info/WHEEL +4 -0
- underscope-0.1.0.dist-info/entry_points.txt +3 -0
underscope/cli.py
ADDED
|
@@ -0,0 +1,2016 @@
|
|
|
1
|
+
"""underscope: see and check a repo's real Python imports.
|
|
2
|
+
|
|
3
|
+
Build one whole-repo Python import graph with ast (stdlib only) and
|
|
4
|
+
render it as mermaid views, one per directory, with a markdown link list
|
|
5
|
+
to zoom in/out. Nodes are file paths (or directory paths for namespace
|
|
6
|
+
packages without __init__.py); edge labels count distinct (importer file,
|
|
7
|
+
target file) pairs.
|
|
8
|
+
A non-ancestor candidate's root must lack __init__.py only when the
|
|
9
|
+
candidate itself is a single file (package candidates skip that filter).
|
|
10
|
+
Test directories (tests/test) and test files (test_*.py, *_test.py,
|
|
11
|
+
conftest.py) are skipped by default, no flag. `import a.b.c` must match
|
|
12
|
+
exactly; `from a.b import c` tries submodule `a.b.c` then falls back to
|
|
13
|
+
`a.b` exactly, never shorter. Each view opens with a summary: cycles are
|
|
14
|
+
strongly connected components (size >= 2) among the view's inside children
|
|
15
|
+
on the view's own edges, outside (dashed) nodes excluded; the node table
|
|
16
|
+
lists one row per inside child. Both sections carry their own legend text
|
|
17
|
+
in the rendered output defining each count; see that legend for units and
|
|
18
|
+
for the SCC_FULL_EDGE_LIMIT threshold controlling full vs. collapsed
|
|
19
|
+
one-way-edge lists in Cycles.
|
|
20
|
+
|
|
21
|
+
WARNING, blind spot: the graph shows import coupling only, a lower bound on
|
|
22
|
+
logical coupling ("if A changes, must B?"). Coupling that bypasses Python
|
|
23
|
+
imports stays invisible: HTTP APIs, shared database schemas and tables,
|
|
24
|
+
config keys and env vars, message and file formats, string-keyed registries,
|
|
25
|
+
runtime sys.path edits. Two services with no edge here can still force each
|
|
26
|
+
other's edits. Never read a missing edge as independence.
|
|
27
|
+
|
|
28
|
+
Three subcommands:
|
|
29
|
+
|
|
30
|
+
`underscope graph <repo> --out DIR [--html]` writes the views under DIR.
|
|
31
|
+
`--html` also writes an index.html per view (mermaid diagram, clickable
|
|
32
|
+
directory nodes) alongside index.md; the browser needs network access to
|
|
33
|
+
load mermaid from a CDN.
|
|
34
|
+
|
|
35
|
+
`underscope check <repo>` runs two convention checks and prints findings to
|
|
36
|
+
stdout, exit 1 on any finding: (1) no import cycles, at every directory
|
|
37
|
+
level (any SCC with more than 1 node in any directory's view, computed the
|
|
38
|
+
same way the view's own Cycles section is, TYPE_CHECKING and
|
|
39
|
+
function-body imports counted, tests excluded as always; no baseline,
|
|
40
|
+
existing cycles fail); (2) a Go-`internal/`-style privacy convention,
|
|
41
|
+
applied to both modules and names (Rust's visibility rule on PEP 8
|
|
42
|
+
underscores): a private item is visible in the thing that defines it and
|
|
43
|
+
that thing's descendants, never in its parent or siblings. For modules: a
|
|
44
|
+
file whose path, relative to the nearest ancestor directory with a
|
|
45
|
+
pyproject.toml [project] table, has a component starting with a single `_`
|
|
46
|
+
(not a dunder) may only be imported by files under the directory holding
|
|
47
|
+
that component, repo-wide, across package boundaries. For names: `from m
|
|
48
|
+
import _x` (or `_x as y`, absolute or relative) where `_x` is not itself a
|
|
49
|
+
resolvable submodule of `m` (that case is the module rule above, not
|
|
50
|
+
double-counted here) -- if `m` resolves to a single file, `_x` is private
|
|
51
|
+
to that file and no other file may import it, including a sibling in the
|
|
52
|
+
same directory or that directory's own `__init__.py`; if `m` resolves to a
|
|
53
|
+
package (`__init__.py` or a namespace package directory), `_x` is private
|
|
54
|
+
to that directory and only files under it may import it. The attribute
|
|
55
|
+
form (`import a.b; a.b._x`) is out of scope here; ruff's SLF001 already
|
|
56
|
+
flags it.
|
|
57
|
+
|
|
58
|
+
`underscope metrics <repo>` prints a coupling report to stdout on the same
|
|
59
|
+
file graph (reach and score per file and per directory, interface width
|
|
60
|
+
per file and per directory, exposure share, entry files, misplaced files)
|
|
61
|
+
and always exits 0: it is a report, never a gate."""
|
|
62
|
+
|
|
63
|
+
import argparse
|
|
64
|
+
import ast
|
|
65
|
+
import html
|
|
66
|
+
import itertools
|
|
67
|
+
import os
|
|
68
|
+
import sys
|
|
69
|
+
import tomllib
|
|
70
|
+
from collections.abc import Callable
|
|
71
|
+
from dataclasses import dataclass
|
|
72
|
+
from pathlib import Path
|
|
73
|
+
|
|
74
|
+
# dotted module path components, e.g. ("shop", "billing", "pricing")
|
|
75
|
+
PathParts = tuple[str, ...]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True, slots=True)
|
|
79
|
+
class ModuleCandidate:
|
|
80
|
+
"""A resolvable module: its file/dir path, and whether it's a namespace
|
|
81
|
+
package directory (True) or a single .py file / __init__.py (False)."""
|
|
82
|
+
|
|
83
|
+
path: Path
|
|
84
|
+
is_namespace_dir: bool
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
ModuleCandidates = dict[PathParts, ModuleCandidate]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass(frozen=True, slots=True)
|
|
91
|
+
class SuffixEntry:
|
|
92
|
+
"""A module candidate indexed by dotted suffix."""
|
|
93
|
+
|
|
94
|
+
comp: PathParts
|
|
95
|
+
path: Path
|
|
96
|
+
is_namespace_dir: bool
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
SuffixIndex = dict[PathParts, list[SuffixEntry]]
|
|
100
|
+
# an ambiguous-import warning: the importing file and its candidate targets
|
|
101
|
+
AmbiguityWarning = tuple[Path, list[Path]]
|
|
102
|
+
# (importer_rel, target_rel) file-level import edge
|
|
103
|
+
EdgeKey = tuple[Path, Path]
|
|
104
|
+
EdgeNames = dict[EdgeKey, set[str]]
|
|
105
|
+
# a resolved absolute/relative import: target file, imported names (if any)
|
|
106
|
+
ResolvedImport = tuple[Path, frozenset[str]]
|
|
107
|
+
# (view node src, view node tgt)
|
|
108
|
+
PairKey = tuple[Path, Path]
|
|
109
|
+
PairEdges = dict[PairKey, set[EdgeKey]]
|
|
110
|
+
# (link text, relative link target) for the markdown "Links:" list
|
|
111
|
+
Link = tuple[str, str]
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@dataclass(frozen=True, slots=True)
|
|
115
|
+
class ScoredCandidate:
|
|
116
|
+
"""A module candidate scored for disambiguation against the importer."""
|
|
117
|
+
|
|
118
|
+
root_depth: int
|
|
119
|
+
is_ancestor: bool
|
|
120
|
+
path: Path
|
|
121
|
+
root: PathParts
|
|
122
|
+
is_namespace_dir: bool
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
SKIP_DIRS = {
|
|
126
|
+
".git",
|
|
127
|
+
".venv",
|
|
128
|
+
"venv",
|
|
129
|
+
"node_modules",
|
|
130
|
+
"__pycache__",
|
|
131
|
+
"build",
|
|
132
|
+
"dist",
|
|
133
|
+
"tests",
|
|
134
|
+
"test",
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def should_skip_dir(name: str) -> bool:
|
|
139
|
+
return name in SKIP_DIRS or name.startswith(".")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def is_test_file(name: str) -> bool:
|
|
143
|
+
return name == "conftest.py" or name.startswith("test_") or name.endswith("_test.py")
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def find_py_files(repo: Path) -> list[Path]:
|
|
147
|
+
files: list[Path] = []
|
|
148
|
+
for dirpath, dirnames, filenames in os.walk(repo):
|
|
149
|
+
dirnames[:] = [d for d in dirnames if not should_skip_dir(d)]
|
|
150
|
+
for fn in filenames:
|
|
151
|
+
if fn.endswith(".py") and not is_test_file(fn):
|
|
152
|
+
files.append(Path(dirpath) / fn)
|
|
153
|
+
return files
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def module_comp(parts: tuple[str, ...]) -> PathParts:
|
|
157
|
+
if parts[-1] == "__init__.py":
|
|
158
|
+
return parts[:-1]
|
|
159
|
+
return parts[:-1] + (parts[-1][:-3],)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def build_index(
|
|
163
|
+
repo: Path, py_files: list[Path]
|
|
164
|
+
) -> tuple[set[Path], ModuleCandidates, SuffixIndex]:
|
|
165
|
+
dirs_with_py: set[Path] = set()
|
|
166
|
+
for f in py_files:
|
|
167
|
+
d = f.relative_to(repo).parent
|
|
168
|
+
while True:
|
|
169
|
+
dirs_with_py.add(d)
|
|
170
|
+
if d == Path("."):
|
|
171
|
+
break
|
|
172
|
+
d = d.parent
|
|
173
|
+
|
|
174
|
+
module_candidates: ModuleCandidates = {}
|
|
175
|
+
for f in py_files:
|
|
176
|
+
rel = f.relative_to(repo)
|
|
177
|
+
comp = module_comp(rel.parts)
|
|
178
|
+
module_candidates[comp] = ModuleCandidate(f, False)
|
|
179
|
+
|
|
180
|
+
for d in dirs_with_py:
|
|
181
|
+
comp = () if d == Path(".") else d.parts
|
|
182
|
+
if comp not in module_candidates:
|
|
183
|
+
module_candidates[comp] = ModuleCandidate(
|
|
184
|
+
repo / d if d != Path(".") else repo, True
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
suffix_index: SuffixIndex = {}
|
|
188
|
+
for comp, cand in module_candidates.items():
|
|
189
|
+
for k in range(1, len(comp) + 1):
|
|
190
|
+
suf = comp[-k:]
|
|
191
|
+
suffix_index.setdefault(suf, []).append(
|
|
192
|
+
SuffixEntry(comp, cand.path, cand.is_namespace_dir)
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
return dirs_with_py, module_candidates, suffix_index
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def path_components(p: Path) -> PathParts:
|
|
199
|
+
return () if p == Path(".") else p.parts
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def disambiguate(
|
|
203
|
+
cands: list[SuffixEntry],
|
|
204
|
+
importer_dir_parts: PathParts,
|
|
205
|
+
suf: PathParts,
|
|
206
|
+
importer_rel: Path,
|
|
207
|
+
warnings: list[AmbiguityWarning],
|
|
208
|
+
module_candidates: ModuleCandidates,
|
|
209
|
+
) -> Path | None:
|
|
210
|
+
uniq: dict[tuple[PathParts, Path], bool] = {}
|
|
211
|
+
for entry in cands:
|
|
212
|
+
uniq[(entry.comp, entry.path)] = entry.is_namespace_dir
|
|
213
|
+
entries = [SuffixEntry(comp, path, isd) for (comp, path), isd in uniq.items()]
|
|
214
|
+
|
|
215
|
+
scored: list[ScoredCandidate] = []
|
|
216
|
+
for entry in entries:
|
|
217
|
+
root = entry.comp[: len(entry.comp) - len(suf)]
|
|
218
|
+
is_ancestor = root == importer_dir_parts[: len(root)]
|
|
219
|
+
scored.append(
|
|
220
|
+
ScoredCandidate(
|
|
221
|
+
len(root), is_ancestor, entry.path, root, entry.is_namespace_dir
|
|
222
|
+
)
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
ancestors = [s for s in scored if s.is_ancestor]
|
|
226
|
+
if ancestors:
|
|
227
|
+
best_depth = max(s.root_depth for s in ancestors)
|
|
228
|
+
best = [s for s in ancestors if s.root_depth == best_depth]
|
|
229
|
+
return best[0].path
|
|
230
|
+
|
|
231
|
+
# a non-ancestor root is only valid if it is not itself inside a package:
|
|
232
|
+
# a module nested in someone else's package is not importable as a
|
|
233
|
+
# top-level name from another project. This filter only applies to
|
|
234
|
+
# single-file candidates (x.py); package candidates (x/__init__.py or a
|
|
235
|
+
# namespace dir x/) skip it.
|
|
236
|
+
def is_package(root: PathParts) -> bool:
|
|
237
|
+
cand = module_candidates.get(root)
|
|
238
|
+
return (
|
|
239
|
+
cand is not None
|
|
240
|
+
and not cand.is_namespace_dir
|
|
241
|
+
and cand.path.name == "__init__.py"
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
def is_single_file_candidate(path: Path, isd: bool) -> bool:
|
|
245
|
+
return not isd and path.name != "__init__.py"
|
|
246
|
+
|
|
247
|
+
non_package_root = [
|
|
248
|
+
s
|
|
249
|
+
for s in scored
|
|
250
|
+
if not (
|
|
251
|
+
is_single_file_candidate(s.path, s.is_namespace_dir) and is_package(s.root)
|
|
252
|
+
)
|
|
253
|
+
]
|
|
254
|
+
if len(non_package_root) == 1:
|
|
255
|
+
return non_package_root[0].path
|
|
256
|
+
|
|
257
|
+
if not non_package_root:
|
|
258
|
+
return None
|
|
259
|
+
|
|
260
|
+
warnings.append((importer_rel, [s.path for s in non_package_root]))
|
|
261
|
+
return None
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def resolve_exact(
|
|
265
|
+
dotted_parts: list[str],
|
|
266
|
+
importer_dir_parts: PathParts,
|
|
267
|
+
suffix_index: SuffixIndex,
|
|
268
|
+
importer_rel: Path,
|
|
269
|
+
warnings: list[AmbiguityWarning],
|
|
270
|
+
module_candidates: ModuleCandidates,
|
|
271
|
+
) -> Path | None:
|
|
272
|
+
if not dotted_parts or dotted_parts[0] in sys.stdlib_module_names:
|
|
273
|
+
return None
|
|
274
|
+
suf = tuple(dotted_parts)
|
|
275
|
+
cands = suffix_index.get(suf)
|
|
276
|
+
if not cands:
|
|
277
|
+
return None
|
|
278
|
+
return disambiguate(
|
|
279
|
+
cands,
|
|
280
|
+
importer_dir_parts,
|
|
281
|
+
suf,
|
|
282
|
+
importer_rel,
|
|
283
|
+
warnings,
|
|
284
|
+
module_candidates,
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def resolve_relative(
|
|
289
|
+
module: str | None,
|
|
290
|
+
level: int,
|
|
291
|
+
names: list[ast.alias],
|
|
292
|
+
importer_dir_parts: PathParts,
|
|
293
|
+
module_candidates: ModuleCandidates,
|
|
294
|
+
) -> list[ResolvedImport]:
|
|
295
|
+
drop = level - 1
|
|
296
|
+
if drop > len(importer_dir_parts):
|
|
297
|
+
return []
|
|
298
|
+
target_dir = (
|
|
299
|
+
importer_dir_parts[: len(importer_dir_parts) - drop]
|
|
300
|
+
if drop
|
|
301
|
+
else importer_dir_parts
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
results: list[ResolvedImport] = []
|
|
305
|
+
if module:
|
|
306
|
+
comp = target_dir + tuple(module.split("."))
|
|
307
|
+
cand = module_candidates.get(comp)
|
|
308
|
+
if cand:
|
|
309
|
+
names_here = frozenset(alias.name for alias in names)
|
|
310
|
+
results.append((cand.path, names_here))
|
|
311
|
+
else:
|
|
312
|
+
for alias in names:
|
|
313
|
+
comp = target_dir + (alias.name,)
|
|
314
|
+
cand = module_candidates.get(comp)
|
|
315
|
+
if cand:
|
|
316
|
+
results.append((cand.path, frozenset()))
|
|
317
|
+
return results
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def top_level_defined_names(tree: ast.Module) -> set[str]:
|
|
321
|
+
"""Names this module defines at top level: def, async def, class, and
|
|
322
|
+
plain or annotated assignment to a bare name. Imports don't count."""
|
|
323
|
+
defined: set[str] = set()
|
|
324
|
+
for stmt in tree.body:
|
|
325
|
+
if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
326
|
+
defined.add(stmt.name)
|
|
327
|
+
elif isinstance(stmt, ast.Assign):
|
|
328
|
+
for target in stmt.targets:
|
|
329
|
+
if isinstance(target, ast.Name):
|
|
330
|
+
defined.add(target.id)
|
|
331
|
+
elif isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name):
|
|
332
|
+
defined.add(stmt.target.id)
|
|
333
|
+
return defined
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def collect_edges(
|
|
337
|
+
repo: Path,
|
|
338
|
+
py_files: list[Path],
|
|
339
|
+
module_candidates: ModuleCandidates,
|
|
340
|
+
suffix_index: SuffixIndex,
|
|
341
|
+
) -> tuple[EdgeNames, list[AmbiguityWarning], dict[Path, set[str]]]:
|
|
342
|
+
# (importer_rel, target_rel) -> set of imported names (empty = no names,
|
|
343
|
+
# e.g. a plain "import x.y" or a "from a import b" where b is itself the
|
|
344
|
+
# resolved submodule rather than a name defined inside it)
|
|
345
|
+
edge_names: EdgeNames = {}
|
|
346
|
+
warnings: list[AmbiguityWarning] = []
|
|
347
|
+
defined_names: dict[Path, set[str]] = {}
|
|
348
|
+
for f in sorted(py_files):
|
|
349
|
+
rel = f.relative_to(repo)
|
|
350
|
+
try:
|
|
351
|
+
text = f.read_text(encoding="utf-8")
|
|
352
|
+
tree = ast.parse(text, filename=str(f))
|
|
353
|
+
except (SyntaxError, UnicodeDecodeError) as e:
|
|
354
|
+
print(f"warning: skipping {rel}: {e}", file=sys.stderr)
|
|
355
|
+
continue
|
|
356
|
+
|
|
357
|
+
defined_names[rel] = top_level_defined_names(tree)
|
|
358
|
+
importer_dir_parts = path_components(rel.parent)
|
|
359
|
+
targets: list[ResolvedImport] = []
|
|
360
|
+
for node in ast.walk(tree):
|
|
361
|
+
if isinstance(node, ast.Import):
|
|
362
|
+
for alias in node.names:
|
|
363
|
+
dotted = alias.name.split(".")
|
|
364
|
+
t = resolve_exact(
|
|
365
|
+
dotted,
|
|
366
|
+
importer_dir_parts,
|
|
367
|
+
suffix_index,
|
|
368
|
+
rel,
|
|
369
|
+
warnings,
|
|
370
|
+
module_candidates,
|
|
371
|
+
)
|
|
372
|
+
if t is not None:
|
|
373
|
+
targets.append((t, frozenset()))
|
|
374
|
+
elif isinstance(node, ast.ImportFrom):
|
|
375
|
+
if node.level and node.level > 0:
|
|
376
|
+
targets.extend(
|
|
377
|
+
resolve_relative(
|
|
378
|
+
node.module,
|
|
379
|
+
node.level,
|
|
380
|
+
node.names,
|
|
381
|
+
importer_dir_parts,
|
|
382
|
+
module_candidates,
|
|
383
|
+
)
|
|
384
|
+
)
|
|
385
|
+
else:
|
|
386
|
+
module_parts = node.module.split(".") if node.module else []
|
|
387
|
+
for alias in node.names:
|
|
388
|
+
t = resolve_exact(
|
|
389
|
+
module_parts + [alias.name],
|
|
390
|
+
importer_dir_parts,
|
|
391
|
+
suffix_index,
|
|
392
|
+
rel,
|
|
393
|
+
warnings,
|
|
394
|
+
module_candidates,
|
|
395
|
+
)
|
|
396
|
+
if t is not None:
|
|
397
|
+
targets.append((t, frozenset()))
|
|
398
|
+
continue
|
|
399
|
+
t = resolve_exact(
|
|
400
|
+
module_parts,
|
|
401
|
+
importer_dir_parts,
|
|
402
|
+
suffix_index,
|
|
403
|
+
rel,
|
|
404
|
+
warnings,
|
|
405
|
+
module_candidates,
|
|
406
|
+
)
|
|
407
|
+
if t is not None:
|
|
408
|
+
targets.append((t, frozenset({alias.name})))
|
|
409
|
+
|
|
410
|
+
for t, names_here in targets:
|
|
411
|
+
trel = t.relative_to(repo)
|
|
412
|
+
if trel == rel:
|
|
413
|
+
continue
|
|
414
|
+
edge_names.setdefault((rel, trel), set()).update(names_here)
|
|
415
|
+
|
|
416
|
+
return edge_names, warnings, defined_names
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def lca(a: Path, b: Path) -> Path:
|
|
420
|
+
pa = () if a == Path(".") else a.parts
|
|
421
|
+
pb = () if b == Path(".") else b.parts
|
|
422
|
+
common: list[str] = []
|
|
423
|
+
for x, y in zip(pa, pb):
|
|
424
|
+
if x != y:
|
|
425
|
+
break
|
|
426
|
+
common.append(x)
|
|
427
|
+
return Path(*common) if common else Path(".")
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def child_of(directory: Path, path: Path) -> Path | None:
|
|
431
|
+
if path == directory:
|
|
432
|
+
return None
|
|
433
|
+
if directory == Path("."):
|
|
434
|
+
return Path(path.parts[0])
|
|
435
|
+
rel = path.relative_to(directory)
|
|
436
|
+
return directory / rel.parts[0]
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def outside_node(directory: Path, path: Path) -> Path | None:
|
|
440
|
+
return child_of(lca(directory, path), path)
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def build_view(directory: Path, edge_names: EdgeNames) -> PairEdges:
|
|
444
|
+
# (node_src, node_tgt) -> set of (importer_rel, target_rel) file-level edges
|
|
445
|
+
pair_edges: PairEdges = {}
|
|
446
|
+
for f_rel, t_rel in edge_names:
|
|
447
|
+
f_in = f_rel.is_relative_to(directory)
|
|
448
|
+
t_in = t_rel.is_relative_to(directory)
|
|
449
|
+
if not f_in and not t_in:
|
|
450
|
+
continue
|
|
451
|
+
if f_in and t_in:
|
|
452
|
+
src = child_of(directory, f_rel)
|
|
453
|
+
tgt = child_of(directory, t_rel)
|
|
454
|
+
elif f_in:
|
|
455
|
+
src = child_of(directory, f_rel)
|
|
456
|
+
tgt = outside_node(directory, t_rel)
|
|
457
|
+
else:
|
|
458
|
+
src = outside_node(directory, f_rel)
|
|
459
|
+
tgt = child_of(directory, t_rel)
|
|
460
|
+
if src is None or tgt is None or src == tgt:
|
|
461
|
+
continue
|
|
462
|
+
pair_edges.setdefault((src, tgt), set()).add((f_rel, t_rel))
|
|
463
|
+
return pair_edges
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def mermaid_id(path: Path) -> str:
|
|
467
|
+
s = str(path).replace("\\", "/")
|
|
468
|
+
return "n_" + "".join(c if c.isalnum() else "_" for c in s)
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def view_index_path(out_dir: Path, directory: Path, name: str = "index.md") -> Path:
|
|
472
|
+
if directory == Path("."):
|
|
473
|
+
return out_dir / name
|
|
474
|
+
return out_dir / directory / name
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
@dataclass(frozen=True, slots=True)
|
|
478
|
+
class MutualPair:
|
|
479
|
+
"""Two SCC members with edges in both directions, and each direction's
|
|
480
|
+
distinct (importer file, target file) pair count."""
|
|
481
|
+
|
|
482
|
+
a: Path
|
|
483
|
+
b: Path
|
|
484
|
+
a_to_b_files: int
|
|
485
|
+
b_to_a_files: int
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
@dataclass(frozen=True, slots=True)
|
|
489
|
+
class SccEdge:
|
|
490
|
+
"""One directed edge inside a strongly connected component."""
|
|
491
|
+
|
|
492
|
+
src: Path
|
|
493
|
+
tgt: Path
|
|
494
|
+
files: int
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
@dataclass(frozen=True, slots=True)
|
|
498
|
+
class Scc:
|
|
499
|
+
"""A strongly connected component (size >= 2) among a view's inside
|
|
500
|
+
children, on the edges the view draws."""
|
|
501
|
+
|
|
502
|
+
members: list[Path]
|
|
503
|
+
mutual_pairs: list[MutualPair]
|
|
504
|
+
edges: list[SccEdge]
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
@dataclass(frozen=True, slots=True)
|
|
508
|
+
class NodeRow:
|
|
509
|
+
"""One inside child's row in a view's summary node table."""
|
|
510
|
+
|
|
511
|
+
node: Path
|
|
512
|
+
importing_units: int
|
|
513
|
+
distinct_names: int
|
|
514
|
+
top_names: list[tuple[str, int]]
|
|
515
|
+
internal: int
|
|
516
|
+
outgoing: int
|
|
517
|
+
root: int | None
|
|
518
|
+
internals: int | None
|
|
519
|
+
private: int
|
|
520
|
+
reach_a: int
|
|
521
|
+
reach_b: int
|
|
522
|
+
score: int
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
def tarjan_sccs(nodes: set[Path], adj: dict[Path, set[Path]]) -> list[list[Path]]:
|
|
526
|
+
"""Iterative Tarjan (explicit stack, no recursion) so large views can't
|
|
527
|
+
hit the recursion limit."""
|
|
528
|
+
counter = 0
|
|
529
|
+
index: dict[Path, int] = {}
|
|
530
|
+
lowlink: dict[Path, int] = {}
|
|
531
|
+
on_stack: dict[Path, bool] = {}
|
|
532
|
+
stack: list[Path] = []
|
|
533
|
+
sccs: list[list[Path]] = []
|
|
534
|
+
|
|
535
|
+
for start in sorted(nodes, key=str):
|
|
536
|
+
if start in index:
|
|
537
|
+
continue
|
|
538
|
+
work: list[tuple[Path, list[Path], int]] = [
|
|
539
|
+
(start, sorted(adj.get(start, ()), key=str), 0)
|
|
540
|
+
]
|
|
541
|
+
index[start] = lowlink[start] = counter
|
|
542
|
+
counter += 1
|
|
543
|
+
stack.append(start)
|
|
544
|
+
on_stack[start] = True
|
|
545
|
+
|
|
546
|
+
while work:
|
|
547
|
+
v, children, pos = work[-1]
|
|
548
|
+
advanced = False
|
|
549
|
+
while pos < len(children):
|
|
550
|
+
w = children[pos]
|
|
551
|
+
pos += 1
|
|
552
|
+
if w not in index:
|
|
553
|
+
index[w] = lowlink[w] = counter
|
|
554
|
+
counter += 1
|
|
555
|
+
stack.append(w)
|
|
556
|
+
on_stack[w] = True
|
|
557
|
+
work[-1] = (v, children, pos)
|
|
558
|
+
work.append((w, sorted(adj.get(w, ()), key=str), 0))
|
|
559
|
+
advanced = True
|
|
560
|
+
break
|
|
561
|
+
if on_stack.get(w, False):
|
|
562
|
+
lowlink[v] = min(lowlink[v], index[w])
|
|
563
|
+
if advanced:
|
|
564
|
+
continue
|
|
565
|
+
work[-1] = (v, children, pos)
|
|
566
|
+
work.pop()
|
|
567
|
+
if work:
|
|
568
|
+
parent = work[-1][0]
|
|
569
|
+
lowlink[parent] = min(lowlink[parent], lowlink[v])
|
|
570
|
+
if lowlink[v] == index[v]:
|
|
571
|
+
comp: list[Path] = []
|
|
572
|
+
while True:
|
|
573
|
+
w = stack.pop()
|
|
574
|
+
on_stack[w] = False
|
|
575
|
+
comp.append(w)
|
|
576
|
+
if w == v:
|
|
577
|
+
break
|
|
578
|
+
sccs.append(comp)
|
|
579
|
+
return sccs
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def compute_sccs(inside_nodes: set[Path], pair_edges: PairEdges) -> list[Scc]:
|
|
583
|
+
adj: dict[Path, set[Path]] = {n: set() for n in inside_nodes}
|
|
584
|
+
file_counts: dict[EdgeKey, int] = {}
|
|
585
|
+
for (src, tgt), file_pairs in pair_edges.items():
|
|
586
|
+
if src in inside_nodes and tgt in inside_nodes:
|
|
587
|
+
adj[src].add(tgt)
|
|
588
|
+
file_counts[(src, tgt)] = len(file_pairs)
|
|
589
|
+
|
|
590
|
+
raw = [comp for comp in tarjan_sccs(inside_nodes, adj) if len(comp) >= 2]
|
|
591
|
+
|
|
592
|
+
result: list[Scc] = []
|
|
593
|
+
for comp in raw:
|
|
594
|
+
members = sorted(comp, key=str)
|
|
595
|
+
member_set = set(comp)
|
|
596
|
+
mutual_pairs: list[MutualPair] = []
|
|
597
|
+
seen: set[frozenset[Path]] = set()
|
|
598
|
+
for a in members:
|
|
599
|
+
for b in members:
|
|
600
|
+
if a == b:
|
|
601
|
+
continue
|
|
602
|
+
if (a, b) in file_counts and (b, a) in file_counts:
|
|
603
|
+
key = frozenset((a, b))
|
|
604
|
+
if key in seen:
|
|
605
|
+
continue
|
|
606
|
+
seen.add(key)
|
|
607
|
+
mutual_pairs.append(
|
|
608
|
+
MutualPair(a, b, file_counts[(a, b)], file_counts[(b, a)])
|
|
609
|
+
)
|
|
610
|
+
mutual_pairs.sort(key=lambda mp: (str(mp.a), str(mp.b)))
|
|
611
|
+
edges = sorted(
|
|
612
|
+
(
|
|
613
|
+
SccEdge(s, t, c)
|
|
614
|
+
for (s, t), c in file_counts.items()
|
|
615
|
+
if s in member_set and t in member_set
|
|
616
|
+
),
|
|
617
|
+
key=lambda e: (str(e.src), str(e.tgt)),
|
|
618
|
+
)
|
|
619
|
+
result.append(Scc(members, mutual_pairs, edges))
|
|
620
|
+
|
|
621
|
+
result.sort(key=lambda scc: (-len(scc.members), str(scc.members[0])))
|
|
622
|
+
return result
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def is_private_part(name: str) -> bool:
|
|
626
|
+
return name.startswith("_") and not name.startswith("__")
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
def has_project_table(pyproject: Path) -> bool:
|
|
630
|
+
try:
|
|
631
|
+
with pyproject.open("rb") as f:
|
|
632
|
+
data = tomllib.load(f)
|
|
633
|
+
except (OSError, tomllib.TOMLDecodeError):
|
|
634
|
+
return False
|
|
635
|
+
return "project" in data
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
def find_package_roots(repo: Path) -> list[Path]:
|
|
639
|
+
"""Directories (relative to repo) whose pyproject.toml has a [project]
|
|
640
|
+
table, used only to anchor the privacy rule's "package root". Sorted
|
|
641
|
+
deepest-first so the closest ancestor is found first."""
|
|
642
|
+
roots: list[Path] = []
|
|
643
|
+
for dirpath, dirnames, filenames in os.walk(repo):
|
|
644
|
+
dirnames[:] = [d for d in dirnames if not should_skip_dir(d)]
|
|
645
|
+
if "pyproject.toml" in filenames and has_project_table(
|
|
646
|
+
Path(dirpath) / "pyproject.toml"
|
|
647
|
+
):
|
|
648
|
+
roots.append(Path(dirpath).relative_to(repo))
|
|
649
|
+
return sorted(roots, key=lambda p: len(p.parts), reverse=True)
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
def owning_package_root(rel: Path, package_roots: list[Path]) -> Path:
|
|
653
|
+
for root in package_roots:
|
|
654
|
+
if root != Path(".") and rel.is_relative_to(root):
|
|
655
|
+
return root
|
|
656
|
+
return Path(".")
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
def private_owner_dir(rel: Path, package_roots: list[Path]) -> Path | None:
|
|
660
|
+
"""The directory a private file/dir component makes it private to, or
|
|
661
|
+
None if no component of rel (relative to its package root) is private."""
|
|
662
|
+
root = owning_package_root(rel, package_roots)
|
|
663
|
+
sub = rel.relative_to(root) if root != Path(".") else rel
|
|
664
|
+
comp = module_comp(sub.parts)
|
|
665
|
+
for k, part in enumerate(comp):
|
|
666
|
+
if is_private_part(part):
|
|
667
|
+
return root / Path(*comp[:k]) if comp[:k] else root
|
|
668
|
+
return None
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
@dataclass(frozen=True, slots=True)
|
|
672
|
+
class PrivacyViolation:
|
|
673
|
+
"""An import that reaches a module private to a directory it is not
|
|
674
|
+
under."""
|
|
675
|
+
|
|
676
|
+
importer: Path
|
|
677
|
+
target: Path
|
|
678
|
+
owner: Path
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
def find_privacy_violations(
|
|
682
|
+
edge_names: EdgeNames, package_roots: list[Path]
|
|
683
|
+
) -> list[PrivacyViolation]:
|
|
684
|
+
violations: list[PrivacyViolation] = []
|
|
685
|
+
owner_cache: dict[Path, Path | None] = {}
|
|
686
|
+
|
|
687
|
+
def owner_of(rel: Path) -> Path | None:
|
|
688
|
+
if rel not in owner_cache:
|
|
689
|
+
owner_cache[rel] = private_owner_dir(rel, package_roots)
|
|
690
|
+
return owner_cache[rel]
|
|
691
|
+
|
|
692
|
+
for importer_rel, target_rel in edge_names:
|
|
693
|
+
owner = owner_of(target_rel)
|
|
694
|
+
if owner is None or importer_rel.is_relative_to(owner):
|
|
695
|
+
continue
|
|
696
|
+
violations.append(PrivacyViolation(importer_rel, target_rel, owner))
|
|
697
|
+
|
|
698
|
+
violations.sort(key=lambda v: (str(v.importer), str(v.target)))
|
|
699
|
+
return violations
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
def is_private_name(name: str) -> bool:
|
|
703
|
+
"""A name private by Rust's convention, ported to PEP 8 underscores:
|
|
704
|
+
one leading underscore, not a dunder, not the bare `_` placeholder
|
|
705
|
+
name."""
|
|
706
|
+
return name.startswith("_") and not name.startswith("__") and name != "_"
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
def name_owner(target_rel: Path) -> tuple[Path, bool]:
|
|
710
|
+
"""Where a name imported `from target_rel import _x` is private to:
|
|
711
|
+
(owner, owner_is_dir). A package (`__init__.py` file, or a namespace
|
|
712
|
+
package directory with no `__init__.py`) makes its names visible to
|
|
713
|
+
files under that directory; a single module file makes its names
|
|
714
|
+
visible to no other file at all."""
|
|
715
|
+
if target_rel.name == "__init__.py":
|
|
716
|
+
return target_rel.parent, True
|
|
717
|
+
if target_rel.suffix != ".py":
|
|
718
|
+
return target_rel, True
|
|
719
|
+
return target_rel, False
|
|
720
|
+
|
|
721
|
+
|
|
722
|
+
@dataclass(frozen=True, slots=True)
|
|
723
|
+
class NameViolation:
|
|
724
|
+
"""An import of a private name (`from m import _x`) that reaches
|
|
725
|
+
outside the directory (package) or file that name is private to."""
|
|
726
|
+
|
|
727
|
+
importer: Path
|
|
728
|
+
target: Path
|
|
729
|
+
name: str
|
|
730
|
+
owner: Path
|
|
731
|
+
owner_is_dir: bool
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def find_name_violations(edge_names: EdgeNames) -> list[NameViolation]:
|
|
735
|
+
violations: list[NameViolation] = []
|
|
736
|
+
for (importer_rel, target_rel), names in edge_names.items():
|
|
737
|
+
for name in names:
|
|
738
|
+
if not is_private_name(name):
|
|
739
|
+
continue
|
|
740
|
+
owner, owner_is_dir = name_owner(target_rel)
|
|
741
|
+
if owner_is_dir and importer_rel.is_relative_to(owner):
|
|
742
|
+
continue
|
|
743
|
+
violations.append(
|
|
744
|
+
NameViolation(importer_rel, target_rel, name, owner, owner_is_dir)
|
|
745
|
+
)
|
|
746
|
+
violations.sort(key=lambda v: (str(v.importer), str(v.target), v.name))
|
|
747
|
+
return violations
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
@dataclass(frozen=True, slots=True)
|
|
751
|
+
class CycleFinding:
|
|
752
|
+
"""One SCC (size >= 2) found in one directory's view, with a concrete
|
|
753
|
+
example loop through file-level edges."""
|
|
754
|
+
|
|
755
|
+
directory: Path
|
|
756
|
+
members: list[Path]
|
|
757
|
+
example: list[Path]
|
|
758
|
+
example_files: list[tuple[Path, Path]]
|
|
759
|
+
|
|
760
|
+
|
|
761
|
+
def find_example_cycle(members: list[Path], pair_edges: PairEdges) -> list[Path]:
|
|
762
|
+
"""One concrete node loop (first == last) through the SCC, via DFS."""
|
|
763
|
+
member_set = set(members)
|
|
764
|
+
adj: dict[Path, list[Path]] = {m: [] for m in members}
|
|
765
|
+
for src, tgt in pair_edges:
|
|
766
|
+
if src in member_set and tgt in member_set and src != tgt:
|
|
767
|
+
adj[src].append(tgt)
|
|
768
|
+
for neighbors in adj.values():
|
|
769
|
+
neighbors.sort(key=str)
|
|
770
|
+
|
|
771
|
+
start = members[0]
|
|
772
|
+
stack: list[Path] = [start]
|
|
773
|
+
visited: set[Path] = {start}
|
|
774
|
+
|
|
775
|
+
def dfs(node: Path) -> list[Path] | None:
|
|
776
|
+
for nxt in adj[node]:
|
|
777
|
+
if nxt == start and len(stack) > 1:
|
|
778
|
+
return [*stack, start]
|
|
779
|
+
if nxt not in visited:
|
|
780
|
+
visited.add(nxt)
|
|
781
|
+
stack.append(nxt)
|
|
782
|
+
result = dfs(nxt)
|
|
783
|
+
if result is not None:
|
|
784
|
+
return result
|
|
785
|
+
stack.pop()
|
|
786
|
+
return None
|
|
787
|
+
|
|
788
|
+
return dfs(start) or [start]
|
|
789
|
+
|
|
790
|
+
|
|
791
|
+
def example_file_edges(
|
|
792
|
+
example: list[Path], pair_edges: PairEdges
|
|
793
|
+
) -> list[tuple[Path, Path]]:
|
|
794
|
+
out: list[tuple[Path, Path]] = []
|
|
795
|
+
for a, b in itertools.pairwise(example):
|
|
796
|
+
file_pairs = pair_edges.get((a, b))
|
|
797
|
+
if file_pairs:
|
|
798
|
+
out.append(min(file_pairs, key=lambda p: (str(p[0]), str(p[1]))))
|
|
799
|
+
return out
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
def compute_inside_nodes(directory: Path, pair_edges: PairEdges) -> set[Path]:
|
|
803
|
+
nodes: set[Path] = set()
|
|
804
|
+
for src, tgt in pair_edges:
|
|
805
|
+
nodes.add(src)
|
|
806
|
+
nodes.add(tgt)
|
|
807
|
+
outside = {n for n in nodes if not n.is_relative_to(directory)}
|
|
808
|
+
return nodes - outside
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
def find_cycle_findings(
|
|
812
|
+
directory: Path, inside_nodes: set[Path], pair_edges: PairEdges
|
|
813
|
+
) -> list[CycleFinding]:
|
|
814
|
+
findings: list[CycleFinding] = []
|
|
815
|
+
for scc in compute_sccs(inside_nodes, pair_edges):
|
|
816
|
+
example = find_example_cycle(scc.members, pair_edges)
|
|
817
|
+
findings.append(
|
|
818
|
+
CycleFinding(
|
|
819
|
+
directory, scc.members, example, example_file_edges(example, pair_edges)
|
|
820
|
+
)
|
|
821
|
+
)
|
|
822
|
+
return findings
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
def count_file_level_cycles(
|
|
826
|
+
py_files: list[Path], repo: Path, edge_names: EdgeNames
|
|
827
|
+
) -> int:
|
|
828
|
+
"""Distinct cycles on the raw file-level graph, ignoring any directory
|
|
829
|
+
view's aggregation -- the number of genuinely separate root causes."""
|
|
830
|
+
nodes = {f.relative_to(repo) for f in py_files}
|
|
831
|
+
adj: dict[Path, set[Path]] = {n: set() for n in nodes}
|
|
832
|
+
for f_rel, t_rel in edge_names:
|
|
833
|
+
if f_rel in adj and t_rel in adj:
|
|
834
|
+
adj[f_rel].add(t_rel)
|
|
835
|
+
sccs = tarjan_sccs(nodes, adj)
|
|
836
|
+
return sum(1 for comp in sccs if len(comp) >= 2)
|
|
837
|
+
|
|
838
|
+
|
|
839
|
+
def build_file_graph(
|
|
840
|
+
nodes: set[Path], edge_names: EdgeNames
|
|
841
|
+
) -> tuple[dict[Path, set[Path]], dict[Path, set[Path]]]:
|
|
842
|
+
"""The raw file-level import graph (forward adjacency and its reverse),
|
|
843
|
+
same edges `--check`'s cycle rule uses: no self-edges (collect_edges
|
|
844
|
+
already drops those), duplicates already collapsed by edge_names'
|
|
845
|
+
dict keys."""
|
|
846
|
+
adj: dict[Path, set[Path]] = {n: set() for n in nodes}
|
|
847
|
+
radj: dict[Path, set[Path]] = {n: set() for n in nodes}
|
|
848
|
+
for importer_rel, target_rel in edge_names:
|
|
849
|
+
if importer_rel in adj and target_rel in adj and importer_rel != target_rel:
|
|
850
|
+
adj[importer_rel].add(target_rel)
|
|
851
|
+
radj[target_rel].add(importer_rel)
|
|
852
|
+
return adj, radj
|
|
853
|
+
|
|
854
|
+
|
|
855
|
+
def reachable_set(start: Path, adj: dict[Path, set[Path]]) -> set[Path]:
|
|
856
|
+
"""Every other node `start` can reach, following `adj` (BFS/DFS, order
|
|
857
|
+
doesn't matter); `start` itself never counts even inside a cycle back
|
|
858
|
+
to itself."""
|
|
859
|
+
seen: set[Path] = set()
|
|
860
|
+
stack = [start]
|
|
861
|
+
while stack:
|
|
862
|
+
node = stack.pop()
|
|
863
|
+
for nxt in adj.get(node, ()):
|
|
864
|
+
if nxt != start and nxt not in seen:
|
|
865
|
+
seen.add(nxt)
|
|
866
|
+
stack.append(nxt)
|
|
867
|
+
return seen
|
|
868
|
+
|
|
869
|
+
|
|
870
|
+
def compute_reach(
|
|
871
|
+
nodes: set[Path], adj: dict[Path, set[Path]], radj: dict[Path, set[Path]]
|
|
872
|
+
) -> tuple[dict[Path, set[Path]], dict[Path, set[Path]]]:
|
|
873
|
+
"""Per file: the set of dependents (other files with an import path to
|
|
874
|
+
it) and the set of dependencies (other files it has an import path
|
|
875
|
+
to), both transitive."""
|
|
876
|
+
dependents = {n: reachable_set(n, radj) for n in nodes}
|
|
877
|
+
dependencies = {n: reachable_set(n, adj) for n in nodes}
|
|
878
|
+
return dependents, dependencies
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
def top_n_by_count(counts: dict[Path, int], n: int = 10) -> list[tuple[Path, int]]:
|
|
882
|
+
return sorted(counts.items(), key=lambda kv: (-kv[1], str(kv[0])))[:n]
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
@dataclass(frozen=True, slots=True)
|
|
886
|
+
class FileScore:
|
|
887
|
+
"""A file's score: dependents x dependencies, both transitive."""
|
|
888
|
+
|
|
889
|
+
path: Path
|
|
890
|
+
dependents: int
|
|
891
|
+
dependencies: int
|
|
892
|
+
score: int
|
|
893
|
+
|
|
894
|
+
|
|
895
|
+
@dataclass(frozen=True, slots=True)
|
|
896
|
+
class DirScore:
|
|
897
|
+
"""A directory's score, measured across its own boundary, repo-wide:
|
|
898
|
+
`reach_in` is how many files outside the directory reach some file
|
|
899
|
+
inside it (transitively); `reach_out` is how many files outside it
|
|
900
|
+
are reached by some file inside it (transitively); score = in x out."""
|
|
901
|
+
|
|
902
|
+
path: Path
|
|
903
|
+
reach_in: int
|
|
904
|
+
reach_out: int
|
|
905
|
+
score: int
|
|
906
|
+
|
|
907
|
+
|
|
908
|
+
def compute_file_scores(
|
|
909
|
+
nodes: set[Path],
|
|
910
|
+
dependents: dict[Path, set[Path]],
|
|
911
|
+
dependencies: dict[Path, set[Path]],
|
|
912
|
+
) -> dict[Path, FileScore]:
|
|
913
|
+
scores: dict[Path, FileScore] = {}
|
|
914
|
+
for n in nodes:
|
|
915
|
+
d_in = len(dependents[n])
|
|
916
|
+
d_out = len(dependencies[n])
|
|
917
|
+
scores[n] = FileScore(n, d_in, d_out, d_in * d_out)
|
|
918
|
+
return scores
|
|
919
|
+
|
|
920
|
+
|
|
921
|
+
def compute_dir_scores(
|
|
922
|
+
dirs_with_py: set[Path],
|
|
923
|
+
dependents: dict[Path, set[Path]],
|
|
924
|
+
dependencies: dict[Path, set[Path]],
|
|
925
|
+
) -> dict[Path, DirScore]:
|
|
926
|
+
dir_in: dict[Path, set[Path]] = {}
|
|
927
|
+
dir_out: dict[Path, set[Path]] = {}
|
|
928
|
+
for f, dep_set in dependents.items():
|
|
929
|
+
for d in ancestor_dirs(f):
|
|
930
|
+
if d in dirs_with_py:
|
|
931
|
+
dir_in.setdefault(d, set()).update(dep_set)
|
|
932
|
+
for f, dep_set in dependencies.items():
|
|
933
|
+
for d in ancestor_dirs(f):
|
|
934
|
+
if d in dirs_with_py:
|
|
935
|
+
dir_out.setdefault(d, set()).update(dep_set)
|
|
936
|
+
|
|
937
|
+
scores: dict[Path, DirScore] = {}
|
|
938
|
+
for d in dirs_with_py:
|
|
939
|
+
in_count = sum(1 for g in dir_in.get(d, set()) if not g.is_relative_to(d))
|
|
940
|
+
out_count = sum(1 for g in dir_out.get(d, set()) if not g.is_relative_to(d))
|
|
941
|
+
scores[d] = DirScore(d, in_count, out_count, in_count * out_count)
|
|
942
|
+
return scores
|
|
943
|
+
|
|
944
|
+
|
|
945
|
+
@dataclass(frozen=True, slots=True)
|
|
946
|
+
class FileWidth:
|
|
947
|
+
"""A target file's interface width: distinct names other files import
|
|
948
|
+
from it, and distinct files importing any of those names."""
|
|
949
|
+
|
|
950
|
+
target: Path
|
|
951
|
+
names: int
|
|
952
|
+
importers: int
|
|
953
|
+
|
|
954
|
+
|
|
955
|
+
def compute_file_widths(nodes: set[Path], edge_names: EdgeNames) -> list[FileWidth]:
|
|
956
|
+
names_by_target: dict[Path, set[str]] = {}
|
|
957
|
+
bare_targets: set[Path] = set()
|
|
958
|
+
importers_by_target: dict[Path, set[Path]] = {}
|
|
959
|
+
for (importer_rel, target_rel), names in edge_names.items():
|
|
960
|
+
if importer_rel == target_rel or target_rel not in nodes:
|
|
961
|
+
continue
|
|
962
|
+
importers_by_target.setdefault(target_rel, set()).add(importer_rel)
|
|
963
|
+
if names:
|
|
964
|
+
names_by_target.setdefault(target_rel, set()).update(names)
|
|
965
|
+
else:
|
|
966
|
+
bare_targets.add(target_rel)
|
|
967
|
+
|
|
968
|
+
widths: list[FileWidth] = []
|
|
969
|
+
for target in nodes:
|
|
970
|
+
name_count = len(names_by_target.get(target, set())) + (
|
|
971
|
+
1 if target in bare_targets else 0
|
|
972
|
+
)
|
|
973
|
+
importer_count = len(importers_by_target.get(target, set()))
|
|
974
|
+
widths.append(FileWidth(target, name_count, importer_count))
|
|
975
|
+
return widths
|
|
976
|
+
|
|
977
|
+
|
|
978
|
+
def ancestor_dirs(rel: Path) -> list[Path]:
|
|
979
|
+
"""The directory holding `rel` and every ancestor directory up to the
|
|
980
|
+
repo root ('.'), inclusive."""
|
|
981
|
+
out = [rel.parent]
|
|
982
|
+
while out[-1] != Path("."):
|
|
983
|
+
out.append(out[-1].parent)
|
|
984
|
+
return out
|
|
985
|
+
|
|
986
|
+
|
|
987
|
+
def compute_wrapper_dirs(nodes: set[Path], dirs_with_py: set[Path]) -> set[Path]:
|
|
988
|
+
"""Directories that hold no .py file of their own and have exactly one
|
|
989
|
+
child directory with Python code, so their file set is identical to
|
|
990
|
+
that child's (and, up a wrapper chain, to the file set of every other
|
|
991
|
+
wrapper in the chain). `--metrics`'s directory tables skip these,
|
|
992
|
+
keeping only the deepest directory of each such chain."""
|
|
993
|
+
direct_file_count: dict[Path, int] = {}
|
|
994
|
+
for f in nodes:
|
|
995
|
+
d = f.parent
|
|
996
|
+
direct_file_count[d] = direct_file_count.get(d, 0) + 1
|
|
997
|
+
|
|
998
|
+
children: dict[Path, list[Path]] = {}
|
|
999
|
+
for d in dirs_with_py:
|
|
1000
|
+
if d == Path("."):
|
|
1001
|
+
continue
|
|
1002
|
+
children.setdefault(d.parent, []).append(d)
|
|
1003
|
+
|
|
1004
|
+
return {
|
|
1005
|
+
d
|
|
1006
|
+
for d in dirs_with_py
|
|
1007
|
+
if direct_file_count.get(d, 0) == 0 and len(children.get(d, [])) == 1
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
|
|
1011
|
+
@dataclass(frozen=True, slots=True)
|
|
1012
|
+
class DirWidth:
|
|
1013
|
+
"""A directory's interface width: distinct (target file, name) pairs
|
|
1014
|
+
files outside it import from files inside it, and distinct outside
|
|
1015
|
+
importing files."""
|
|
1016
|
+
|
|
1017
|
+
directory: Path
|
|
1018
|
+
names: int
|
|
1019
|
+
importers: int
|
|
1020
|
+
|
|
1021
|
+
|
|
1022
|
+
def compute_dir_widths(dirs_with_py: set[Path], edge_names: EdgeNames) -> list[DirWidth]:
|
|
1023
|
+
named_pairs: dict[Path, set[tuple[Path, str]]] = {}
|
|
1024
|
+
bare_targets: dict[Path, set[Path]] = {}
|
|
1025
|
+
importers: dict[Path, set[Path]] = {}
|
|
1026
|
+
for (importer_rel, target_rel), names in edge_names.items():
|
|
1027
|
+
for directory in ancestor_dirs(target_rel):
|
|
1028
|
+
if directory not in dirs_with_py:
|
|
1029
|
+
continue
|
|
1030
|
+
if importer_rel.is_relative_to(directory):
|
|
1031
|
+
continue
|
|
1032
|
+
importers.setdefault(directory, set()).add(importer_rel)
|
|
1033
|
+
if names:
|
|
1034
|
+
named_pairs.setdefault(directory, set()).update(
|
|
1035
|
+
(target_rel, name) for name in names
|
|
1036
|
+
)
|
|
1037
|
+
else:
|
|
1038
|
+
bare_targets.setdefault(directory, set()).add(target_rel)
|
|
1039
|
+
|
|
1040
|
+
widths: list[DirWidth] = []
|
|
1041
|
+
for directory in dirs_with_py:
|
|
1042
|
+
name_count = len(named_pairs.get(directory, set())) + len(
|
|
1043
|
+
bare_targets.get(directory, set())
|
|
1044
|
+
)
|
|
1045
|
+
importer_count = len(importers.get(directory, set()))
|
|
1046
|
+
widths.append(DirWidth(directory, name_count, importer_count))
|
|
1047
|
+
return widths
|
|
1048
|
+
|
|
1049
|
+
|
|
1050
|
+
@dataclass(frozen=True, slots=True)
|
|
1051
|
+
class Exposure:
|
|
1052
|
+
"""A file's exposure share: how many of its top-level defined names
|
|
1053
|
+
(def, async def, class, plain/annotated assignment to a bare name;
|
|
1054
|
+
imports don't count) at least one other file actually imports."""
|
|
1055
|
+
|
|
1056
|
+
path: Path
|
|
1057
|
+
used: int
|
|
1058
|
+
defined: int
|
|
1059
|
+
share: float
|
|
1060
|
+
importers: int
|
|
1061
|
+
|
|
1062
|
+
|
|
1063
|
+
def compute_exposure(
|
|
1064
|
+
nodes: set[Path], edge_names: EdgeNames, defined_names: dict[Path, set[str]]
|
|
1065
|
+
) -> dict[Path, Exposure]:
|
|
1066
|
+
imported_names: dict[Path, set[str]] = {}
|
|
1067
|
+
importers_by_name: dict[Path, dict[str, set[Path]]] = {}
|
|
1068
|
+
for (importer_rel, target_rel), names in edge_names.items():
|
|
1069
|
+
if not names or importer_rel == target_rel:
|
|
1070
|
+
continue
|
|
1071
|
+
imported_names.setdefault(target_rel, set()).update(names)
|
|
1072
|
+
per_name = importers_by_name.setdefault(target_rel, {})
|
|
1073
|
+
for name in names:
|
|
1074
|
+
per_name.setdefault(name, set()).add(importer_rel)
|
|
1075
|
+
|
|
1076
|
+
exposures: dict[Path, Exposure] = {}
|
|
1077
|
+
for f in nodes:
|
|
1078
|
+
if f.name == "__init__.py":
|
|
1079
|
+
continue
|
|
1080
|
+
defined = defined_names.get(f, set())
|
|
1081
|
+
used_names = defined & imported_names.get(f, set())
|
|
1082
|
+
per_name = importers_by_name.get(f, {})
|
|
1083
|
+
importer_set: set[Path] = set()
|
|
1084
|
+
for name in used_names:
|
|
1085
|
+
importer_set.update(per_name.get(name, set()))
|
|
1086
|
+
used = len(used_names)
|
|
1087
|
+
total = len(defined)
|
|
1088
|
+
share = used / total if total else 0.0
|
|
1089
|
+
exposures[f] = Exposure(f, used, total, share, len(importer_set))
|
|
1090
|
+
return exposures
|
|
1091
|
+
|
|
1092
|
+
|
|
1093
|
+
@dataclass(frozen=True, slots=True)
|
|
1094
|
+
class EntryFiles:
|
|
1095
|
+
"""A directory's entry-file share: how many of its files at least one
|
|
1096
|
+
file outside the directory imports directly, out of all files inside
|
|
1097
|
+
it; outside_importers is the distinct outside files doing so."""
|
|
1098
|
+
|
|
1099
|
+
directory: Path
|
|
1100
|
+
entry: int
|
|
1101
|
+
files: int
|
|
1102
|
+
share: float
|
|
1103
|
+
outside_importers: int
|
|
1104
|
+
|
|
1105
|
+
|
|
1106
|
+
def compute_entry_files(
|
|
1107
|
+
dirs_with_py: set[Path], nodes: set[Path], edge_names: EdgeNames
|
|
1108
|
+
) -> dict[Path, EntryFiles]:
|
|
1109
|
+
entry_by_dir: dict[Path, set[Path]] = {}
|
|
1110
|
+
outside_importers_by_dir: dict[Path, set[Path]] = {}
|
|
1111
|
+
for importer_rel, target_rel in edge_names:
|
|
1112
|
+
for directory in ancestor_dirs(target_rel):
|
|
1113
|
+
if directory not in dirs_with_py:
|
|
1114
|
+
continue
|
|
1115
|
+
if importer_rel.is_relative_to(directory):
|
|
1116
|
+
continue
|
|
1117
|
+
entry_by_dir.setdefault(directory, set()).add(target_rel)
|
|
1118
|
+
outside_importers_by_dir.setdefault(directory, set()).add(importer_rel)
|
|
1119
|
+
|
|
1120
|
+
files_by_dir: dict[Path, int] = {}
|
|
1121
|
+
for f in nodes:
|
|
1122
|
+
for directory in ancestor_dirs(f):
|
|
1123
|
+
if directory in dirs_with_py:
|
|
1124
|
+
files_by_dir[directory] = files_by_dir.get(directory, 0) + 1
|
|
1125
|
+
|
|
1126
|
+
out: dict[Path, EntryFiles] = {}
|
|
1127
|
+
for directory in dirs_with_py:
|
|
1128
|
+
files = files_by_dir.get(directory, 0)
|
|
1129
|
+
entry = len(entry_by_dir.get(directory, set()))
|
|
1130
|
+
share = entry / files if files else 0.0
|
|
1131
|
+
outside = len(outside_importers_by_dir.get(directory, set()))
|
|
1132
|
+
out[directory] = EntryFiles(directory, entry, files, share, outside)
|
|
1133
|
+
return out
|
|
1134
|
+
|
|
1135
|
+
|
|
1136
|
+
@dataclass(frozen=True, slots=True)
|
|
1137
|
+
class Misplaced:
|
|
1138
|
+
"""A file whose direct import partners (files it imports plus files
|
|
1139
|
+
importing it, from the same file graph the scores use) cluster more
|
|
1140
|
+
heavily in some other directory than in its own."""
|
|
1141
|
+
|
|
1142
|
+
path: Path
|
|
1143
|
+
own: int
|
|
1144
|
+
other: int
|
|
1145
|
+
other_dir: Path
|
|
1146
|
+
imported_by: int
|
|
1147
|
+
imports: int
|
|
1148
|
+
|
|
1149
|
+
|
|
1150
|
+
def compute_misplaced(
|
|
1151
|
+
nodes: set[Path],
|
|
1152
|
+
adj: dict[Path, set[Path]],
|
|
1153
|
+
radj: dict[Path, set[Path]],
|
|
1154
|
+
) -> list[Misplaced]:
|
|
1155
|
+
results: list[Misplaced] = []
|
|
1156
|
+
for f in nodes:
|
|
1157
|
+
importers = radj.get(f, set()) - {f}
|
|
1158
|
+
imports_of = adj.get(f, set()) - {f}
|
|
1159
|
+
partners = importers | imports_of
|
|
1160
|
+
own_dir = f.parent
|
|
1161
|
+
own = 0
|
|
1162
|
+
other_counts: dict[Path, int] = {}
|
|
1163
|
+
for p in partners:
|
|
1164
|
+
d = p.parent
|
|
1165
|
+
if d == own_dir:
|
|
1166
|
+
own += 1
|
|
1167
|
+
else:
|
|
1168
|
+
other_counts[d] = other_counts.get(d, 0) + 1
|
|
1169
|
+
other = 0
|
|
1170
|
+
other_dir = Path(".")
|
|
1171
|
+
if other_counts:
|
|
1172
|
+
best = max(other_counts.values())
|
|
1173
|
+
# tie-break deterministically: smallest path among the ties
|
|
1174
|
+
other_dir = min(d for d, c in other_counts.items() if c == best)
|
|
1175
|
+
other = best
|
|
1176
|
+
if other >= 3 and other > own:
|
|
1177
|
+
imported_by = sum(1 for p in importers if p.parent == other_dir)
|
|
1178
|
+
imports = sum(1 for p in imports_of if p.parent == other_dir)
|
|
1179
|
+
results.append(Misplaced(f, own, other, other_dir, imported_by, imports))
|
|
1180
|
+
return results
|
|
1181
|
+
|
|
1182
|
+
|
|
1183
|
+
def render_metrics(
|
|
1184
|
+
repo: Path,
|
|
1185
|
+
py_files: list[Path],
|
|
1186
|
+
dirs_with_py: set[Path],
|
|
1187
|
+
edge_names: EdgeNames,
|
|
1188
|
+
file_scores: dict[Path, FileScore],
|
|
1189
|
+
dir_scores: dict[Path, DirScore],
|
|
1190
|
+
defined_names: dict[Path, set[str]],
|
|
1191
|
+
adj: dict[Path, set[Path]],
|
|
1192
|
+
radj: dict[Path, set[Path]],
|
|
1193
|
+
) -> list[str]:
|
|
1194
|
+
nodes = {f.relative_to(repo) for f in py_files}
|
|
1195
|
+
n = len(nodes)
|
|
1196
|
+
dependents = {p: fs.dependents for p, fs in file_scores.items()}
|
|
1197
|
+
dependencies = {p: fs.dependencies for p, fs in file_scores.items()}
|
|
1198
|
+
wrapper_dirs = compute_wrapper_dirs(nodes, dirs_with_py)
|
|
1199
|
+
|
|
1200
|
+
lines: list[str] = ["# Coupling report", ""]
|
|
1201
|
+
lines.append("## Reach")
|
|
1202
|
+
lines.append("")
|
|
1203
|
+
lines.append(
|
|
1204
|
+
"dependents(f): other files with an import path to f, direct or "
|
|
1205
|
+
+ "transitive; dependencies(f): other files f has an import path "
|
|
1206
|
+
+ "to; score(f) = dependents(f) x dependencies(f)."
|
|
1207
|
+
)
|
|
1208
|
+
lines.append("")
|
|
1209
|
+
lines.append(f"- N (non-test .py files): {n}")
|
|
1210
|
+
lines.append(f"- file-level edges: {len(edge_names)}")
|
|
1211
|
+
lines.append("")
|
|
1212
|
+
|
|
1213
|
+
lines.append("Top 10 by dependents:")
|
|
1214
|
+
lines.append("")
|
|
1215
|
+
for path, count in top_n_by_count(dependents):
|
|
1216
|
+
share = count / n * 100 if n else 0.0
|
|
1217
|
+
lines.append(f"- {path}: {count} ({share:.1f}% of N)")
|
|
1218
|
+
lines.append("")
|
|
1219
|
+
|
|
1220
|
+
lines.append("Top 10 by dependencies:")
|
|
1221
|
+
lines.append("")
|
|
1222
|
+
for path, count in top_n_by_count(dependencies):
|
|
1223
|
+
share = count / n * 100 if n else 0.0
|
|
1224
|
+
lines.append(f"- {path}: {count} ({share:.1f}% of N)")
|
|
1225
|
+
lines.append("")
|
|
1226
|
+
|
|
1227
|
+
lines.append(
|
|
1228
|
+
"All files by score (dependents x dependencies), ties: dependents "
|
|
1229
|
+
+ "desc, then path:"
|
|
1230
|
+
)
|
|
1231
|
+
lines.append("")
|
|
1232
|
+
lines.append("| file | dependents | dependencies | score |")
|
|
1233
|
+
lines.append("| --- | --- | --- | --- |")
|
|
1234
|
+
for fs in sorted(
|
|
1235
|
+
file_scores.values(), key=lambda s: (-s.score, -s.dependents, str(s.path))
|
|
1236
|
+
):
|
|
1237
|
+
lines.append(f"| {fs.path} | {fs.dependents} | {fs.dependencies} | {fs.score} |")
|
|
1238
|
+
lines.append("")
|
|
1239
|
+
|
|
1240
|
+
lines.append(
|
|
1241
|
+
"All directories by score (in x out, measured across the "
|
|
1242
|
+
+ "directory's own boundary, repo-wide); a directory whose file "
|
|
1243
|
+
+ "set is identical to its one child directory's is skipped, "
|
|
1244
|
+
+ "keeping only the deepest directory of such a wrapper chain:"
|
|
1245
|
+
)
|
|
1246
|
+
lines.append("")
|
|
1247
|
+
lines.append("| directory | in | out | score |")
|
|
1248
|
+
lines.append("| --- | --- | --- | --- |")
|
|
1249
|
+
for ds in sorted(
|
|
1250
|
+
(s for s in dir_scores.values() if s.path not in wrapper_dirs),
|
|
1251
|
+
key=lambda s: (-s.score, str(s.path)),
|
|
1252
|
+
):
|
|
1253
|
+
directory_label = "repo root" if ds.path == Path(".") else str(ds.path)
|
|
1254
|
+
lines.append(
|
|
1255
|
+
f"| {directory_label} | {ds.reach_in} | {ds.reach_out} | {ds.score} |"
|
|
1256
|
+
)
|
|
1257
|
+
lines.append("")
|
|
1258
|
+
|
|
1259
|
+
lines.append("## Interface width")
|
|
1260
|
+
lines.append("")
|
|
1261
|
+
lines.append(
|
|
1262
|
+
"distinct names other files import from a file or directory "
|
|
1263
|
+
+ "(a nameless import, e.g. `import x`, counts the target as 1 "
|
|
1264
|
+
+ "name), and the distinct files importing any of them."
|
|
1265
|
+
)
|
|
1266
|
+
lines.append("")
|
|
1267
|
+
|
|
1268
|
+
file_widths = sorted(
|
|
1269
|
+
compute_file_widths(nodes, edge_names),
|
|
1270
|
+
key=lambda w: (-w.names, str(w.target)),
|
|
1271
|
+
)[:10]
|
|
1272
|
+
lines.append("Top 10 files by name count:")
|
|
1273
|
+
lines.append("")
|
|
1274
|
+
for w in file_widths:
|
|
1275
|
+
lines.append(f"- {w.target}: {w.names} names, {w.importers} importers")
|
|
1276
|
+
lines.append("")
|
|
1277
|
+
|
|
1278
|
+
dir_widths = sorted(
|
|
1279
|
+
(
|
|
1280
|
+
w
|
|
1281
|
+
for w in compute_dir_widths(dirs_with_py, edge_names)
|
|
1282
|
+
if w.directory not in wrapper_dirs
|
|
1283
|
+
),
|
|
1284
|
+
key=lambda w: (-w.names, str(w.directory)),
|
|
1285
|
+
)[:10]
|
|
1286
|
+
lines.append(
|
|
1287
|
+
"Top 10 directories by name count (wrapper directories, see " + "above, skipped):"
|
|
1288
|
+
)
|
|
1289
|
+
lines.append("")
|
|
1290
|
+
for w in dir_widths:
|
|
1291
|
+
directory_label = "repo root" if w.directory == Path(".") else str(w.directory)
|
|
1292
|
+
lines.append(f"- {directory_label}: {w.names} names, {w.importers} importers")
|
|
1293
|
+
lines.append("")
|
|
1294
|
+
|
|
1295
|
+
lines.append("## Exposure share")
|
|
1296
|
+
lines.append("")
|
|
1297
|
+
lines.append(
|
|
1298
|
+
"Per file (`__init__.py` excluded): defined = top-level names it "
|
|
1299
|
+
+ "defines (def, async def, class, plain/annotated assignment to a "
|
|
1300
|
+
+ "bare name; imports don't count); used = those names at least one "
|
|
1301
|
+
+ "other file imports; share = used / defined. Files with defined "
|
|
1302
|
+
+ ">= 10, sorted by share desc, then defined desc."
|
|
1303
|
+
)
|
|
1304
|
+
lines.append("")
|
|
1305
|
+
exposures = [
|
|
1306
|
+
e
|
|
1307
|
+
for e in compute_exposure(nodes, edge_names, defined_names).values()
|
|
1308
|
+
if e.defined >= 10
|
|
1309
|
+
]
|
|
1310
|
+
exposures.sort(key=lambda e: (-e.share, -e.defined, str(e.path)))
|
|
1311
|
+
lines.append("| file | used | defined | share | importing files |")
|
|
1312
|
+
lines.append("| --- | --- | --- | --- | --- |")
|
|
1313
|
+
for e in exposures:
|
|
1314
|
+
lines.append(
|
|
1315
|
+
f"| {e.path} | {e.used} | {e.defined} | {e.share * 100:.1f}% "
|
|
1316
|
+
+ f"| {e.importers} |"
|
|
1317
|
+
)
|
|
1318
|
+
lines.append("")
|
|
1319
|
+
|
|
1320
|
+
lines.append("## Entry files")
|
|
1321
|
+
lines.append("")
|
|
1322
|
+
lines.append(
|
|
1323
|
+
"Per directory (directories with >= 5 files): entry files = files "
|
|
1324
|
+
+ "inside the directory at least one file outside it imports "
|
|
1325
|
+
+ "directly; share = entry files / files; outside importers = "
|
|
1326
|
+
+ "distinct outside files importing anything inside. Sorted by "
|
|
1327
|
+
+ "share desc, then outside importers desc. Wrapper directories, "
|
|
1328
|
+
+ "see above, skipped."
|
|
1329
|
+
)
|
|
1330
|
+
lines.append("")
|
|
1331
|
+
entries = [
|
|
1332
|
+
e
|
|
1333
|
+
for e in compute_entry_files(dirs_with_py, nodes, edge_names).values()
|
|
1334
|
+
if e.files >= 5 and e.directory not in wrapper_dirs
|
|
1335
|
+
]
|
|
1336
|
+
entries.sort(key=lambda e: (-e.share, -e.outside_importers, str(e.directory)))
|
|
1337
|
+
lines.append("| directory | entry files | files | share | outside importers |")
|
|
1338
|
+
lines.append("| --- | --- | --- | --- | --- |")
|
|
1339
|
+
for e in entries:
|
|
1340
|
+
directory_label = "repo root" if e.directory == Path(".") else str(e.directory)
|
|
1341
|
+
lines.append(
|
|
1342
|
+
f"| {directory_label} | {e.entry} | {e.files} | {e.share * 100:.1f}% "
|
|
1343
|
+
+ f"| {e.outside_importers} |"
|
|
1344
|
+
)
|
|
1345
|
+
lines.append("")
|
|
1346
|
+
|
|
1347
|
+
lines.append("## Misplaced files")
|
|
1348
|
+
lines.append("")
|
|
1349
|
+
misplaced = sorted(
|
|
1350
|
+
compute_misplaced(nodes, adj, radj),
|
|
1351
|
+
key=lambda m: (-(m.other - m.own), str(m.path)),
|
|
1352
|
+
)
|
|
1353
|
+
lines.append(f"{len(misplaced)} of {n} files misplaced.")
|
|
1354
|
+
lines.append("")
|
|
1355
|
+
lines.append(
|
|
1356
|
+
"partners(f) = files f imports directly plus files importing f "
|
|
1357
|
+
+ "directly; own = partners in f's own parent directory; other = "
|
|
1358
|
+
+ "the largest count among all other parent directories, naming "
|
|
1359
|
+
+ "that directory; misplaced when other >= 3 and other > own. "
|
|
1360
|
+
+ "'imported by' = those other-directory partners that import f; "
|
|
1361
|
+
+ "'imports' = those other-directory partners f imports; a "
|
|
1362
|
+
+ "partner doing both counts in both, so the two may sum to more "
|
|
1363
|
+
+ "than other. Sorted by other - own desc, then path."
|
|
1364
|
+
)
|
|
1365
|
+
lines.append("")
|
|
1366
|
+
lines.append("| file | own | other | imported by | imports | other directory |")
|
|
1367
|
+
lines.append("| --- | --- | --- | --- | --- | --- |")
|
|
1368
|
+
for m in misplaced:
|
|
1369
|
+
other_dir_label = "repo root" if m.other_dir == Path(".") else str(m.other_dir)
|
|
1370
|
+
lines.append(
|
|
1371
|
+
f"| {m.path} | {m.own} | {m.other} | {m.imported_by} | {m.imports} "
|
|
1372
|
+
+ f"| {other_dir_label} |"
|
|
1373
|
+
)
|
|
1374
|
+
lines.append("")
|
|
1375
|
+
|
|
1376
|
+
return lines
|
|
1377
|
+
|
|
1378
|
+
|
|
1379
|
+
def compute_node_rows(
|
|
1380
|
+
repo: Path,
|
|
1381
|
+
inside_nodes: set[Path],
|
|
1382
|
+
dirs_with_py: set[Path],
|
|
1383
|
+
pair_edges: PairEdges,
|
|
1384
|
+
edge_names: EdgeNames,
|
|
1385
|
+
file_scores: dict[Path, FileScore],
|
|
1386
|
+
dir_scores: dict[Path, DirScore],
|
|
1387
|
+
) -> list[NodeRow]:
|
|
1388
|
+
rows: list[NodeRow] = []
|
|
1389
|
+
for node in inside_nodes:
|
|
1390
|
+
incoming_file_pairs: set[EdgeKey] = set()
|
|
1391
|
+
importing_units: set[Path] = set()
|
|
1392
|
+
name_units: dict[str, set[Path]] = {}
|
|
1393
|
+
distinct_target_names: set[tuple[Path, str]] = set()
|
|
1394
|
+
private = 0
|
|
1395
|
+
|
|
1396
|
+
for (src, tgt), file_pairs in pair_edges.items():
|
|
1397
|
+
if tgt != node or src == node:
|
|
1398
|
+
continue
|
|
1399
|
+
importing_units.add(src)
|
|
1400
|
+
incoming_file_pairs |= file_pairs
|
|
1401
|
+
for f_rel, t_rel in file_pairs:
|
|
1402
|
+
for name in edge_names.get((f_rel, t_rel), ()):
|
|
1403
|
+
distinct_target_names.add((t_rel, name))
|
|
1404
|
+
name_units.setdefault(name, set()).add(src)
|
|
1405
|
+
if name.startswith("_") and not name.startswith("__"):
|
|
1406
|
+
private += 1
|
|
1407
|
+
|
|
1408
|
+
top_names = sorted(name_units.items(), key=lambda kv: (-len(kv[1]), kv[0]))[:5]
|
|
1409
|
+
top_names_fmt = [(name, len(units)) for name, units in top_names]
|
|
1410
|
+
|
|
1411
|
+
internal_pairs = {
|
|
1412
|
+
(f_rel, t_rel)
|
|
1413
|
+
for (f_rel, t_rel) in edge_names
|
|
1414
|
+
if f_rel.is_relative_to(node) and t_rel.is_relative_to(node)
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
outgoing_pairs: set[EdgeKey] = set()
|
|
1418
|
+
for (src, tgt), file_pairs in pair_edges.items():
|
|
1419
|
+
if src == node and tgt != node:
|
|
1420
|
+
outgoing_pairs |= file_pairs
|
|
1421
|
+
|
|
1422
|
+
root: int | None
|
|
1423
|
+
internals: int | None
|
|
1424
|
+
init_path = node / "__init__.py"
|
|
1425
|
+
if node in dirs_with_py and (repo / init_path).is_file():
|
|
1426
|
+
root = sum(1 for _, t_rel in incoming_file_pairs if t_rel == init_path)
|
|
1427
|
+
internals = len(incoming_file_pairs) - root
|
|
1428
|
+
else:
|
|
1429
|
+
root = None
|
|
1430
|
+
internals = None
|
|
1431
|
+
|
|
1432
|
+
if node in dirs_with_py:
|
|
1433
|
+
ds = dir_scores[node]
|
|
1434
|
+
reach_a, reach_b, score = ds.reach_in, ds.reach_out, ds.score
|
|
1435
|
+
else:
|
|
1436
|
+
fs = file_scores[node]
|
|
1437
|
+
reach_a, reach_b, score = fs.dependents, fs.dependencies, fs.score
|
|
1438
|
+
|
|
1439
|
+
rows.append(
|
|
1440
|
+
NodeRow(
|
|
1441
|
+
node,
|
|
1442
|
+
len(importing_units),
|
|
1443
|
+
len(distinct_target_names),
|
|
1444
|
+
top_names_fmt,
|
|
1445
|
+
len(internal_pairs),
|
|
1446
|
+
len(outgoing_pairs),
|
|
1447
|
+
root,
|
|
1448
|
+
internals,
|
|
1449
|
+
private,
|
|
1450
|
+
reach_a,
|
|
1451
|
+
reach_b,
|
|
1452
|
+
score,
|
|
1453
|
+
)
|
|
1454
|
+
)
|
|
1455
|
+
|
|
1456
|
+
rows.sort(key=lambda r: (-r.importing_units, str(r.node)))
|
|
1457
|
+
return rows
|
|
1458
|
+
|
|
1459
|
+
|
|
1460
|
+
# an SCC with at most this many internal edges (one per ordered (src, tgt)
|
|
1461
|
+
# child pair, not file pairs) lists every edge; above it, only mutual pairs
|
|
1462
|
+
# are listed plus a count of the rest (see Edges for those).
|
|
1463
|
+
SCC_FULL_EDGE_LIMIT = 30
|
|
1464
|
+
|
|
1465
|
+
|
|
1466
|
+
def render_summary(
|
|
1467
|
+
repo: Path,
|
|
1468
|
+
inside_nodes: set[Path],
|
|
1469
|
+
dirs_with_py: set[Path],
|
|
1470
|
+
pair_edges: PairEdges,
|
|
1471
|
+
edge_names: EdgeNames,
|
|
1472
|
+
label: Callable[[Path], str],
|
|
1473
|
+
file_scores: dict[Path, FileScore],
|
|
1474
|
+
dir_scores: dict[Path, DirScore],
|
|
1475
|
+
) -> list[str]:
|
|
1476
|
+
lines: list[str] = ["## Cycles", ""]
|
|
1477
|
+
sccs = compute_sccs(inside_nodes, pair_edges)
|
|
1478
|
+
if not sccs:
|
|
1479
|
+
lines.append("no cycles")
|
|
1480
|
+
else:
|
|
1481
|
+
for scc in sccs:
|
|
1482
|
+
lines.append(f"- {', '.join(label(m) for m in scc.members)}")
|
|
1483
|
+
for mp in scc.mutual_pairs:
|
|
1484
|
+
pair_str = f"{label(mp.a)} <-> {label(mp.b)}"
|
|
1485
|
+
count_str = f"({mp.a_to_b_files}/{mp.b_to_a_files})"
|
|
1486
|
+
lines.append(f" - {pair_str} {count_str}")
|
|
1487
|
+
mutual_members = {(mp.a, mp.b) for mp in scc.mutual_pairs} | {
|
|
1488
|
+
(mp.b, mp.a) for mp in scc.mutual_pairs
|
|
1489
|
+
}
|
|
1490
|
+
one_way = [e for e in scc.edges if (e.src, e.tgt) not in mutual_members]
|
|
1491
|
+
if not scc.mutual_pairs or len(scc.edges) <= SCC_FULL_EDGE_LIMIT:
|
|
1492
|
+
# A->B->C->A with no mutual pair: the one-way edges are the
|
|
1493
|
+
# cycle itself, so keep the full list instead of collapsing
|
|
1494
|
+
# it. Small SCCs (<= SCC_FULL_EDGE_LIMIT internal edges) also
|
|
1495
|
+
# list every one-way edge instead of collapsing to a count.
|
|
1496
|
+
for e in one_way:
|
|
1497
|
+
lines.append(f" - {label(e.src)} -> {label(e.tgt)} ({e.files})")
|
|
1498
|
+
elif one_way:
|
|
1499
|
+
lines.append(
|
|
1500
|
+
f" - {len(one_way)} more edges inside this cycle, see Edges."
|
|
1501
|
+
)
|
|
1502
|
+
lines.append("")
|
|
1503
|
+
lines.append(
|
|
1504
|
+
"`a <-> b (X/Y)` means X file pairs a->b, Y file pairs b->a "
|
|
1505
|
+
+ "(distinct importer,target file pairs, same count as an Edges heading)."
|
|
1506
|
+
)
|
|
1507
|
+
lines.append(
|
|
1508
|
+
f"An SCC with at most {SCC_FULL_EDGE_LIMIT} internal edges (one "
|
|
1509
|
+
+ "per ordered child pair) lists every one-way edge as "
|
|
1510
|
+
+ "`a -> b (N)`; above that, the one-way edges shrink to a count, "
|
|
1511
|
+
+ "see Edges, unless the SCC has no mutual pair."
|
|
1512
|
+
)
|
|
1513
|
+
lines.append("")
|
|
1514
|
+
|
|
1515
|
+
lines.append("## Node summary")
|
|
1516
|
+
lines.append("")
|
|
1517
|
+
rows = compute_node_rows(
|
|
1518
|
+
repo,
|
|
1519
|
+
inside_nodes,
|
|
1520
|
+
dirs_with_py,
|
|
1521
|
+
pair_edges,
|
|
1522
|
+
edge_names,
|
|
1523
|
+
file_scores,
|
|
1524
|
+
dir_scores,
|
|
1525
|
+
)
|
|
1526
|
+
lines.append(
|
|
1527
|
+
"| node | importing units | names | internal | outgoing | root/internals "
|
|
1528
|
+
+ "| private | dependents/in | dependencies/out | score |"
|
|
1529
|
+
)
|
|
1530
|
+
lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |")
|
|
1531
|
+
for row in rows:
|
|
1532
|
+
names_cell = str(row.distinct_names)
|
|
1533
|
+
if row.top_names:
|
|
1534
|
+
top_str = ", ".join(f"{n} ({c})" for n, c in row.top_names)
|
|
1535
|
+
names_cell = f"{row.distinct_names} ({top_str})"
|
|
1536
|
+
root_cell = "-" if row.root is None else f"{row.root}/{row.internals}"
|
|
1537
|
+
left = f"| {label(row.node)} | {row.importing_units} | {names_cell} |"
|
|
1538
|
+
right = f" {row.internal} | {row.outgoing} | {root_cell} | {row.private} |"
|
|
1539
|
+
reach_cell = f" {row.reach_a} | {row.reach_b} | {row.score} |"
|
|
1540
|
+
lines.append(left + right + reach_cell)
|
|
1541
|
+
lines.append("")
|
|
1542
|
+
lines.append(
|
|
1543
|
+
"- importing units: distinct other view nodes with an edge into this node"
|
|
1544
|
+
)
|
|
1545
|
+
lines.append(
|
|
1546
|
+
"- names: distinct (target file, name) pairs imported from this node, "
|
|
1547
|
+
+ "then the top 5 names by importing-unit count"
|
|
1548
|
+
)
|
|
1549
|
+
lines.append(
|
|
1550
|
+
"- internal: distinct (importer file, target file) pairs with both "
|
|
1551
|
+
+ "files inside this node"
|
|
1552
|
+
)
|
|
1553
|
+
lines.append(
|
|
1554
|
+
"- outgoing: distinct (importer file, target file) pairs from this "
|
|
1555
|
+
+ "node to other view nodes"
|
|
1556
|
+
)
|
|
1557
|
+
lines.append(
|
|
1558
|
+
"- root/internals: incoming file pairs into this node's own "
|
|
1559
|
+
+ '__init__.py vs into its deeper files ("-" for a single file or a '
|
|
1560
|
+
+ "namespace dir without __init__.py)"
|
|
1561
|
+
)
|
|
1562
|
+
lines.append(
|
|
1563
|
+
"- private: (importer, target, name) triples from other nodes where "
|
|
1564
|
+
+ 'name starts with a single "_" (not dunder)'
|
|
1565
|
+
)
|
|
1566
|
+
lines.append(
|
|
1567
|
+
"- dependents/in, dependencies/out, score: computed once repo-wide "
|
|
1568
|
+
+ "(not from this view's edges); for a file node, transitive "
|
|
1569
|
+
+ "dependents and dependencies and their product; for a directory "
|
|
1570
|
+
+ "node, files outside it that reach in/are reached out of it "
|
|
1571
|
+
+ "transitively, and their product"
|
|
1572
|
+
)
|
|
1573
|
+
lines.append("")
|
|
1574
|
+
return lines
|
|
1575
|
+
|
|
1576
|
+
|
|
1577
|
+
MERMAID_CDN_URL = "https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs"
|
|
1578
|
+
# headroom above mermaid's defaults (maxTextSize 50000, maxEdges 500): the
|
|
1579
|
+
# largest view seen in practice had a ~38000-char mermaid block with ~400
|
|
1580
|
+
# edges, so double both defaults.
|
|
1581
|
+
MERMAID_MAX_TEXT_SIZE = 100_000
|
|
1582
|
+
MERMAID_MAX_EDGES = 1_000
|
|
1583
|
+
|
|
1584
|
+
|
|
1585
|
+
def build_mermaid_lines(
|
|
1586
|
+
nodes: set[Path],
|
|
1587
|
+
ids: dict[Path, str],
|
|
1588
|
+
outside_nodes: set[Path],
|
|
1589
|
+
label: Callable[[Path], str],
|
|
1590
|
+
ordered_pairs: list[tuple[PairKey, set[EdgeKey]]],
|
|
1591
|
+
) -> list[str]:
|
|
1592
|
+
lines: list[str] = ["flowchart LR"]
|
|
1593
|
+
lines.append(" classDef outside fill:#eee,stroke:#999,stroke-dasharray: 5 5;")
|
|
1594
|
+
for n in sorted(nodes, key=str):
|
|
1595
|
+
lines.append(f' {ids[n]}["{label(n)}"]')
|
|
1596
|
+
for (src, tgt), file_pairs in ordered_pairs:
|
|
1597
|
+
lines.append(f' {ids[src]} -->|"{len(file_pairs)}"| {ids[tgt]}')
|
|
1598
|
+
if outside_nodes:
|
|
1599
|
+
lines.append(
|
|
1600
|
+
" class " + ",".join(sorted(ids[n] for n in outside_nodes)) + " outside;"
|
|
1601
|
+
)
|
|
1602
|
+
return lines
|
|
1603
|
+
|
|
1604
|
+
|
|
1605
|
+
def write_html_view(
|
|
1606
|
+
out_dir: Path,
|
|
1607
|
+
directory: Path,
|
|
1608
|
+
mermaid_lines: list[str],
|
|
1609
|
+
nodes: set[Path],
|
|
1610
|
+
dirs_with_py: set[Path],
|
|
1611
|
+
) -> None:
|
|
1612
|
+
html_path = view_index_path(out_dir, directory, "index.html")
|
|
1613
|
+
title = "repo root" if directory == Path(".") else str(directory)
|
|
1614
|
+
|
|
1615
|
+
body_lines = list(mermaid_lines)
|
|
1616
|
+
for n in sorted(nodes, key=str):
|
|
1617
|
+
if n in dirs_with_py:
|
|
1618
|
+
target_html = view_index_path(out_dir, n, "index.html")
|
|
1619
|
+
rel_link = os.path.relpath(target_html, start=html_path.parent)
|
|
1620
|
+
body_lines.append(f' click {mermaid_id(n)} "{rel_link}"')
|
|
1621
|
+
mermaid_src = html.escape("\n".join(body_lines))
|
|
1622
|
+
|
|
1623
|
+
parent_link_html = ""
|
|
1624
|
+
if directory != Path("."):
|
|
1625
|
+
parent_index = view_index_path(out_dir, directory.parent, "index.html")
|
|
1626
|
+
rel_link = os.path.relpath(parent_index, start=html_path.parent)
|
|
1627
|
+
parent_link_html = f'<p><a href="{rel_link}">parent view</a></p>\n'
|
|
1628
|
+
|
|
1629
|
+
page = f"""<!DOCTYPE html>
|
|
1630
|
+
<html>
|
|
1631
|
+
<head>
|
|
1632
|
+
<meta charset="utf-8">
|
|
1633
|
+
<title>import graph: {html.escape(title)}</title>
|
|
1634
|
+
<style>
|
|
1635
|
+
pre.mermaid {{ overflow: auto; white-space: pre; }}
|
|
1636
|
+
pre.mermaid svg {{ max-width: none !important; }}
|
|
1637
|
+
</style>
|
|
1638
|
+
</head>
|
|
1639
|
+
<body>
|
|
1640
|
+
<h1>import graph: {html.escape(title)}</h1>
|
|
1641
|
+
{parent_link_html}<pre class="mermaid">
|
|
1642
|
+
{mermaid_src}
|
|
1643
|
+
</pre>
|
|
1644
|
+
<script type="module">
|
|
1645
|
+
import mermaid from "{MERMAID_CDN_URL}";
|
|
1646
|
+
mermaid.initialize({{
|
|
1647
|
+
startOnLoad: true,
|
|
1648
|
+
securityLevel: "loose",
|
|
1649
|
+
maxTextSize: {MERMAID_MAX_TEXT_SIZE},
|
|
1650
|
+
maxEdges: {MERMAID_MAX_EDGES},
|
|
1651
|
+
flowchart: {{ useMaxWidth: false }}
|
|
1652
|
+
}});
|
|
1653
|
+
</script>
|
|
1654
|
+
</body>
|
|
1655
|
+
</html>
|
|
1656
|
+
"""
|
|
1657
|
+
html_path.write_text(page, encoding="utf-8")
|
|
1658
|
+
|
|
1659
|
+
|
|
1660
|
+
def write_view(
|
|
1661
|
+
out_dir: Path,
|
|
1662
|
+
repo: Path,
|
|
1663
|
+
directory: Path,
|
|
1664
|
+
dirs_with_py: set[Path],
|
|
1665
|
+
pair_edges: PairEdges,
|
|
1666
|
+
edge_names: EdgeNames,
|
|
1667
|
+
write_html: bool,
|
|
1668
|
+
file_scores: dict[Path, FileScore],
|
|
1669
|
+
dir_scores: dict[Path, DirScore],
|
|
1670
|
+
) -> None:
|
|
1671
|
+
index_path = view_index_path(out_dir, directory)
|
|
1672
|
+
index_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1673
|
+
|
|
1674
|
+
nodes: set[Path] = set()
|
|
1675
|
+
for src, tgt in pair_edges:
|
|
1676
|
+
nodes.add(src)
|
|
1677
|
+
nodes.add(tgt)
|
|
1678
|
+
|
|
1679
|
+
ids = {n: mermaid_id(n) for n in nodes}
|
|
1680
|
+
outside_nodes: set[Path] = set()
|
|
1681
|
+
for src, tgt in pair_edges:
|
|
1682
|
+
if not src.is_relative_to(directory):
|
|
1683
|
+
outside_nodes.add(src)
|
|
1684
|
+
if not tgt.is_relative_to(directory):
|
|
1685
|
+
outside_nodes.add(tgt)
|
|
1686
|
+
|
|
1687
|
+
def label(n: Path) -> str:
|
|
1688
|
+
return str(n) if n in outside_nodes else n.name
|
|
1689
|
+
|
|
1690
|
+
lines: list[str] = []
|
|
1691
|
+
title = "repo root" if directory == Path(".") else str(directory)
|
|
1692
|
+
lines.append(f"# import graph: {title}")
|
|
1693
|
+
lines.append("")
|
|
1694
|
+
|
|
1695
|
+
links: list[Link] = [] # (text, target_index_path) for nodes that have a view
|
|
1696
|
+
|
|
1697
|
+
if directory != Path("."):
|
|
1698
|
+
parent = directory.parent
|
|
1699
|
+
parent_index = view_index_path(out_dir, parent)
|
|
1700
|
+
rel_link = os.path.relpath(parent_index, start=index_path.parent)
|
|
1701
|
+
lines.append(f"[parent view]({rel_link})")
|
|
1702
|
+
lines.append("")
|
|
1703
|
+
|
|
1704
|
+
inside_nodes = nodes - outside_nodes
|
|
1705
|
+
lines.extend(
|
|
1706
|
+
render_summary(
|
|
1707
|
+
repo,
|
|
1708
|
+
inside_nodes,
|
|
1709
|
+
dirs_with_py,
|
|
1710
|
+
pair_edges,
|
|
1711
|
+
edge_names,
|
|
1712
|
+
label,
|
|
1713
|
+
file_scores,
|
|
1714
|
+
dir_scores,
|
|
1715
|
+
)
|
|
1716
|
+
)
|
|
1717
|
+
|
|
1718
|
+
ordered_pairs = sorted(
|
|
1719
|
+
pair_edges.items(), key=lambda kv: (str(kv[0][0]), str(kv[0][1]))
|
|
1720
|
+
)
|
|
1721
|
+
mermaid_lines = build_mermaid_lines(nodes, ids, outside_nodes, label, ordered_pairs)
|
|
1722
|
+
|
|
1723
|
+
lines.append("```mermaid")
|
|
1724
|
+
lines.extend(mermaid_lines)
|
|
1725
|
+
|
|
1726
|
+
for n in sorted(nodes, key=str):
|
|
1727
|
+
if n in dirs_with_py:
|
|
1728
|
+
target_index = view_index_path(out_dir, n)
|
|
1729
|
+
rel_link = os.path.relpath(target_index, start=index_path.parent)
|
|
1730
|
+
links.append((str(n), rel_link))
|
|
1731
|
+
lines.append("```")
|
|
1732
|
+
lines.append("")
|
|
1733
|
+
|
|
1734
|
+
if links:
|
|
1735
|
+
lines.append("Links:")
|
|
1736
|
+
lines.append("")
|
|
1737
|
+
for text, rel_link in links:
|
|
1738
|
+
lines.append(f"- [{text}]({rel_link})")
|
|
1739
|
+
lines.append("")
|
|
1740
|
+
|
|
1741
|
+
if ordered_pairs:
|
|
1742
|
+
lines.append("Edges:")
|
|
1743
|
+
lines.append("")
|
|
1744
|
+
for (src, tgt), file_pairs in ordered_pairs:
|
|
1745
|
+
unit = "file pair" if len(file_pairs) == 1 else "file pairs"
|
|
1746
|
+
lines.append(f"### {label(src)} -> {label(tgt)} ({len(file_pairs)} {unit})")
|
|
1747
|
+
lines.append("")
|
|
1748
|
+
for f_rel, t_rel in sorted(file_pairs, key=lambda p: (str(p[0]), str(p[1]))):
|
|
1749
|
+
names = sorted(edge_names.get((f_rel, t_rel), ()))
|
|
1750
|
+
if names:
|
|
1751
|
+
lines.append(f"- {f_rel} -> {t_rel}: {', '.join(names)}")
|
|
1752
|
+
else:
|
|
1753
|
+
lines.append(f"- {f_rel} -> {t_rel}")
|
|
1754
|
+
lines.append("")
|
|
1755
|
+
|
|
1756
|
+
index_path.write_text("\n".join(lines), encoding="utf-8")
|
|
1757
|
+
|
|
1758
|
+
if write_html:
|
|
1759
|
+
write_html_view(out_dir, directory, mermaid_lines, nodes, dirs_with_py)
|
|
1760
|
+
|
|
1761
|
+
|
|
1762
|
+
__version__ = "0.1.0"
|
|
1763
|
+
|
|
1764
|
+
|
|
1765
|
+
class CheckArgs(argparse.Namespace):
|
|
1766
|
+
# placeholder default for the static type only; argparse always
|
|
1767
|
+
# overwrites it before parse_args() returns
|
|
1768
|
+
repo: Path = Path()
|
|
1769
|
+
|
|
1770
|
+
|
|
1771
|
+
class GraphArgs(argparse.Namespace):
|
|
1772
|
+
repo: Path = Path()
|
|
1773
|
+
out: Path = Path()
|
|
1774
|
+
html: bool = False
|
|
1775
|
+
|
|
1776
|
+
|
|
1777
|
+
class MetricsArgs(argparse.Namespace):
|
|
1778
|
+
repo: Path = Path()
|
|
1779
|
+
|
|
1780
|
+
|
|
1781
|
+
def build_graph(
|
|
1782
|
+
repo: Path,
|
|
1783
|
+
) -> tuple[
|
|
1784
|
+
list[Path],
|
|
1785
|
+
set[Path],
|
|
1786
|
+
EdgeNames,
|
|
1787
|
+
dict[Path, set[str]],
|
|
1788
|
+
dict[Path, set[Path]],
|
|
1789
|
+
dict[Path, set[Path]],
|
|
1790
|
+
dict[Path, FileScore],
|
|
1791
|
+
dict[Path, DirScore],
|
|
1792
|
+
]:
|
|
1793
|
+
"""Parses `repo` once and builds every structure the subcommands share:
|
|
1794
|
+
the file list, the directories holding Python code, the file-level
|
|
1795
|
+
import edges (and their imported names), each file's top-level defined
|
|
1796
|
+
names, the forward/reverse file adjacency, and the file/directory
|
|
1797
|
+
scores. Prints ambiguous-import warnings to stderr along the way."""
|
|
1798
|
+
py_files = find_py_files(repo)
|
|
1799
|
+
dirs_with_py, module_candidates, suffix_index = build_index(repo, py_files)
|
|
1800
|
+
edge_names, warnings, defined_names = collect_edges(
|
|
1801
|
+
repo, py_files, module_candidates, suffix_index
|
|
1802
|
+
)
|
|
1803
|
+
|
|
1804
|
+
for importer_rel, cands in warnings:
|
|
1805
|
+
cand_strs = ", ".join(str(c.relative_to(repo)) for c in cands)
|
|
1806
|
+
print(
|
|
1807
|
+
f"warning: ambiguous import in {importer_rel}: candidates {cand_strs}",
|
|
1808
|
+
file=sys.stderr,
|
|
1809
|
+
)
|
|
1810
|
+
|
|
1811
|
+
nodes = {f.relative_to(repo) for f in py_files}
|
|
1812
|
+
adj, radj = build_file_graph(nodes, edge_names)
|
|
1813
|
+
dependents, dependencies = compute_reach(nodes, adj, radj)
|
|
1814
|
+
file_scores = compute_file_scores(nodes, dependents, dependencies)
|
|
1815
|
+
dir_scores = compute_dir_scores(dirs_with_py, dependents, dependencies)
|
|
1816
|
+
|
|
1817
|
+
return (
|
|
1818
|
+
py_files,
|
|
1819
|
+
dirs_with_py,
|
|
1820
|
+
edge_names,
|
|
1821
|
+
defined_names,
|
|
1822
|
+
adj,
|
|
1823
|
+
radj,
|
|
1824
|
+
file_scores,
|
|
1825
|
+
dir_scores,
|
|
1826
|
+
)
|
|
1827
|
+
|
|
1828
|
+
|
|
1829
|
+
def print_check_findings(
|
|
1830
|
+
cycle_findings: list[CycleFinding],
|
|
1831
|
+
distinct_file_cycles: int,
|
|
1832
|
+
privacy_violations: list[PrivacyViolation],
|
|
1833
|
+
name_violations: list[NameViolation],
|
|
1834
|
+
) -> bool:
|
|
1835
|
+
"""Prints findings, returns True if any finding exists."""
|
|
1836
|
+
any_finding = (
|
|
1837
|
+
bool(cycle_findings) or bool(privacy_violations) or bool(name_violations)
|
|
1838
|
+
)
|
|
1839
|
+
|
|
1840
|
+
print(
|
|
1841
|
+
f"## Cycles: {len(cycle_findings)} finding(s) across all directory "
|
|
1842
|
+
+ f"views, from {distinct_file_cycles} distinct file-level cycle(s)"
|
|
1843
|
+
)
|
|
1844
|
+
if cycle_findings:
|
|
1845
|
+
print(
|
|
1846
|
+
"(a directory view merges each child directory into one node, "
|
|
1847
|
+
+ "so a view's cycles need not match the file-level cycles: "
|
|
1848
|
+
+ "2 directories can import each other with no file-level cycle, "
|
|
1849
|
+
+ "and several file-level cycles can merge into one)"
|
|
1850
|
+
)
|
|
1851
|
+
for cf in cycle_findings:
|
|
1852
|
+
directory_label = "repo root" if cf.directory == Path(".") else str(cf.directory)
|
|
1853
|
+
print(f"- {directory_label}: {len(cf.members)} nodes")
|
|
1854
|
+
print(f" members: {', '.join(str(m) for m in cf.members)}")
|
|
1855
|
+
print(f" example: {' -> '.join(str(n) for n in cf.example)}")
|
|
1856
|
+
for a, b in cf.example_files:
|
|
1857
|
+
print(f" {a} -> {b}")
|
|
1858
|
+
print()
|
|
1859
|
+
|
|
1860
|
+
total_private = len(privacy_violations) + len(name_violations)
|
|
1861
|
+
print(f"## Private-item violations: {total_private} finding(s)")
|
|
1862
|
+
for v in privacy_violations:
|
|
1863
|
+
print(f"- {v.importer} -> {v.target}: allowed only inside {v.owner}")
|
|
1864
|
+
for nv in name_violations:
|
|
1865
|
+
scope = "directory" if nv.owner_is_dir else "file"
|
|
1866
|
+
print(
|
|
1867
|
+
f"- {nv.importer} -> {nv.target}: name {nv.name} private to "
|
|
1868
|
+
+ f"{scope} {nv.owner}"
|
|
1869
|
+
)
|
|
1870
|
+
print()
|
|
1871
|
+
|
|
1872
|
+
return any_finding
|
|
1873
|
+
|
|
1874
|
+
|
|
1875
|
+
def run_check(repo: Path) -> int:
|
|
1876
|
+
"""`underscope check <repo>`: the import-cycle and privacy-convention
|
|
1877
|
+
checks, findings printed to stdout, exit 1 on any finding, no views
|
|
1878
|
+
written."""
|
|
1879
|
+
py_files, dirs_with_py, edge_names, _, _, _, _, _ = build_graph(repo)
|
|
1880
|
+
|
|
1881
|
+
cycle_findings: list[CycleFinding] = []
|
|
1882
|
+
for directory in sorted(dirs_with_py, key=str):
|
|
1883
|
+
pair_edges = build_view(directory, edge_names)
|
|
1884
|
+
inside_nodes = compute_inside_nodes(directory, pair_edges)
|
|
1885
|
+
cycle_findings.extend(find_cycle_findings(directory, inside_nodes, pair_edges))
|
|
1886
|
+
|
|
1887
|
+
package_roots = find_package_roots(repo)
|
|
1888
|
+
privacy_violations = find_privacy_violations(edge_names, package_roots)
|
|
1889
|
+
name_violations = find_name_violations(edge_names)
|
|
1890
|
+
distinct_file_cycles = count_file_level_cycles(py_files, repo, edge_names)
|
|
1891
|
+
any_finding = print_check_findings(
|
|
1892
|
+
cycle_findings, distinct_file_cycles, privacy_violations, name_violations
|
|
1893
|
+
)
|
|
1894
|
+
return 1 if any_finding else 0
|
|
1895
|
+
|
|
1896
|
+
|
|
1897
|
+
def run_graph(repo: Path, out_dir: Path, write_html: bool) -> int:
|
|
1898
|
+
"""`underscope graph <repo> --out DIR [--html]`: writes one mermaid
|
|
1899
|
+
view per directory under DIR."""
|
|
1900
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
1901
|
+
_, dirs_with_py, edge_names, _, _, _, file_scores, dir_scores = build_graph(repo)
|
|
1902
|
+
|
|
1903
|
+
written = 0
|
|
1904
|
+
for directory in sorted(dirs_with_py, key=str):
|
|
1905
|
+
pair_edges = build_view(directory, edge_names)
|
|
1906
|
+
write_view(
|
|
1907
|
+
out_dir,
|
|
1908
|
+
repo,
|
|
1909
|
+
directory,
|
|
1910
|
+
dirs_with_py,
|
|
1911
|
+
pair_edges,
|
|
1912
|
+
edge_names,
|
|
1913
|
+
write_html,
|
|
1914
|
+
file_scores,
|
|
1915
|
+
dir_scores,
|
|
1916
|
+
)
|
|
1917
|
+
written += 1
|
|
1918
|
+
|
|
1919
|
+
print(f"wrote {written} views under {out_dir}", file=sys.stderr)
|
|
1920
|
+
return 0
|
|
1921
|
+
|
|
1922
|
+
|
|
1923
|
+
def run_metrics(repo: Path) -> int:
|
|
1924
|
+
"""`underscope metrics <repo>`: prints a coupling report to stdout,
|
|
1925
|
+
always exits 0."""
|
|
1926
|
+
(
|
|
1927
|
+
py_files,
|
|
1928
|
+
dirs_with_py,
|
|
1929
|
+
edge_names,
|
|
1930
|
+
defined_names,
|
|
1931
|
+
adj,
|
|
1932
|
+
radj,
|
|
1933
|
+
file_scores,
|
|
1934
|
+
dir_scores,
|
|
1935
|
+
) = build_graph(repo)
|
|
1936
|
+
|
|
1937
|
+
print(
|
|
1938
|
+
"\n".join(
|
|
1939
|
+
render_metrics(
|
|
1940
|
+
repo,
|
|
1941
|
+
py_files,
|
|
1942
|
+
dirs_with_py,
|
|
1943
|
+
edge_names,
|
|
1944
|
+
file_scores,
|
|
1945
|
+
dir_scores,
|
|
1946
|
+
defined_names,
|
|
1947
|
+
adj,
|
|
1948
|
+
radj,
|
|
1949
|
+
)
|
|
1950
|
+
)
|
|
1951
|
+
)
|
|
1952
|
+
return 0
|
|
1953
|
+
|
|
1954
|
+
|
|
1955
|
+
def main() -> int:
|
|
1956
|
+
parser = argparse.ArgumentParser(
|
|
1957
|
+
prog="underscope",
|
|
1958
|
+
description="Check and measure a Python repo's imports: no import cycles, "
|
|
1959
|
+
+ "underscore privacy scoped like Rust's pub(super)/pub(crate), coupling metrics.",
|
|
1960
|
+
epilog="Rules, metrics and blind spots: https://github.com/dertilo/underscope",
|
|
1961
|
+
)
|
|
1962
|
+
parser.add_argument(
|
|
1963
|
+
"--version", action="version", version=f"underscope {__version__}"
|
|
1964
|
+
)
|
|
1965
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
1966
|
+
|
|
1967
|
+
check_parser = subparsers.add_parser(
|
|
1968
|
+
"check",
|
|
1969
|
+
help=(
|
|
1970
|
+
"run the import-cycle and privacy convention checks, print "
|
|
1971
|
+
+ "findings to stdout, exit 1 on any finding"
|
|
1972
|
+
),
|
|
1973
|
+
)
|
|
1974
|
+
check_parser.add_argument("repo", type=Path)
|
|
1975
|
+
|
|
1976
|
+
graph_parser = subparsers.add_parser("graph", help="write mermaid views")
|
|
1977
|
+
graph_parser.add_argument("repo", type=Path)
|
|
1978
|
+
graph_parser.add_argument("--out", type=Path, required=True)
|
|
1979
|
+
graph_parser.add_argument(
|
|
1980
|
+
"--html",
|
|
1981
|
+
action="store_true",
|
|
1982
|
+
help=(
|
|
1983
|
+
"also write index.html per view; needs network access in the "
|
|
1984
|
+
+ "browser to load mermaid from a CDN"
|
|
1985
|
+
),
|
|
1986
|
+
)
|
|
1987
|
+
|
|
1988
|
+
metrics_parser = subparsers.add_parser(
|
|
1989
|
+
"metrics",
|
|
1990
|
+
help=(
|
|
1991
|
+
"print a coupling report (reach and score, interface "
|
|
1992
|
+
+ "width, exposure share, entry files, misplaced files) to "
|
|
1993
|
+
+ "stdout; always exits 0"
|
|
1994
|
+
),
|
|
1995
|
+
)
|
|
1996
|
+
metrics_parser.add_argument("repo", type=Path)
|
|
1997
|
+
|
|
1998
|
+
args = parser.parse_args()
|
|
1999
|
+
|
|
2000
|
+
if args.command == "check":
|
|
2001
|
+
check_args = CheckArgs()
|
|
2002
|
+
check_args.repo = args.repo
|
|
2003
|
+
return run_check(check_args.repo.resolve())
|
|
2004
|
+
if args.command == "graph":
|
|
2005
|
+
graph_args = GraphArgs()
|
|
2006
|
+
graph_args.repo = args.repo
|
|
2007
|
+
graph_args.out = args.out
|
|
2008
|
+
graph_args.html = args.html
|
|
2009
|
+
return run_graph(graph_args.repo.resolve(), graph_args.out, graph_args.html)
|
|
2010
|
+
metrics_args = MetricsArgs()
|
|
2011
|
+
metrics_args.repo = args.repo
|
|
2012
|
+
return run_metrics(metrics_args.repo.resolve())
|
|
2013
|
+
|
|
2014
|
+
|
|
2015
|
+
if __name__ == "__main__":
|
|
2016
|
+
raise SystemExit(main())
|