repotour 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- repotour/__init__.py +5 -0
- repotour/__main__.py +5 -0
- repotour/analyzers/__init__.py +0 -0
- repotour/analyzers/base.py +29 -0
- repotour/analyzers/basics.py +73 -0
- repotour/analyzers/bulk.py +50 -0
- repotour/analyzers/commands.py +301 -0
- repotour/analyzers/dependencies.py +226 -0
- repotour/analyzers/directories.py +224 -0
- repotour/analyzers/entry_points.py +127 -0
- repotour/analyzers/history.py +104 -0
- repotour/analyzers/manifest_entries.py +145 -0
- repotour/analyzers/overview.py +127 -0
- repotour/analyzers/ownership.py +243 -0
- repotour/analyzers/project.py +368 -0
- repotour/analyzers/reading_order.py +213 -0
- repotour/cache/__init__.py +0 -0
- repotour/cache/history.py +66 -0
- repotour/cache/parse.py +46 -0
- repotour/cache/store.py +190 -0
- repotour/cli.py +134 -0
- repotour/config.py +232 -0
- repotour/errors.py +25 -0
- repotour/git/__init__.py +0 -0
- repotour/git/reader.py +300 -0
- repotour/graph/__init__.py +1 -0
- repotour/graph/collapse.py +76 -0
- repotour/graph/pagerank.py +39 -0
- repotour/languages/__init__.py +13 -0
- repotour/languages/base.py +84 -0
- repotour/languages/go.py +162 -0
- repotour/languages/javascript.py +377 -0
- repotour/languages/python.py +300 -0
- repotour/model/__init__.py +0 -0
- repotour/model/history.py +51 -0
- repotour/model/serde.py +66 -0
- repotour/model/snapshot.py +76 -0
- repotour/model/tour.py +338 -0
- repotour/output.py +44 -0
- repotour/pipeline.py +226 -0
- repotour/py.typed +0 -0
- repotour/render/__init__.py +0 -0
- repotour/render/html.py +73 -0
- repotour/render/json.py +29 -0
- repotour/render/markdown.py +23 -0
- repotour/render/mermaid.py +21 -0
- repotour/render/templates/tour.html.j2 +65 -0
- repotour/render/templates/tour.md.j2 +236 -0
- repotour/render/vendor/mermaid-LICENSE +21 -0
- repotour/render/vendor/mermaid.tiny.js +3581 -0
- repotour/repopath.py +38 -0
- repotour/scan/__init__.py +0 -0
- repotour/scan/classifier.py +153 -0
- repotour/scan/filetypes.py +94 -0
- repotour/scan/globs.py +57 -0
- repotour/scan/scanner.py +65 -0
- repotour/summarize/__init__.py +2 -0
- repotour/summarize/provider.py +151 -0
- repotour/summarize/summarizer.py +178 -0
- repotour-1.0.0.dist-info/METADATA +193 -0
- repotour-1.0.0.dist-info/RECORD +64 -0
- repotour-1.0.0.dist-info/WHEEL +4 -0
- repotour-1.0.0.dist-info/entry_points.txt +2 -0
- repotour-1.0.0.dist-info/licenses/LICENSE +21 -0
repotour/__init__.py
ADDED
repotour/__main__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""The analyzer contract. Each analyzer produces one section of the tour."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import Any, Protocol
|
|
7
|
+
|
|
8
|
+
from repotour.model.snapshot import RepoSnapshot
|
|
9
|
+
from repotour.model.tour import Evidence
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class AnalyzerResult:
|
|
14
|
+
analyzer: str
|
|
15
|
+
data: dict[str, Any] # analyzer-specific, JSON-serializable
|
|
16
|
+
evidence: list[Evidence] = field(default_factory=list)
|
|
17
|
+
warnings: list[str] = field(default_factory=list)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Analyzer(Protocol):
|
|
21
|
+
name: str
|
|
22
|
+
version: int # bump to invalidate this analyzer's cache
|
|
23
|
+
depends_on: tuple[str, ...]
|
|
24
|
+
|
|
25
|
+
def run(
|
|
26
|
+
self,
|
|
27
|
+
snapshot: RepoSnapshot,
|
|
28
|
+
inputs: dict[str, AnalyzerResult],
|
|
29
|
+
) -> AnalyzerResult: ...
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Basics a newcomer expects: README, tests, contributing guide, license, CI.
|
|
2
|
+
|
|
3
|
+
Each missing item is a claim whose evidence states what was looked for.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from collections.abc import Callable
|
|
9
|
+
|
|
10
|
+
from repotour import repopath
|
|
11
|
+
from repotour.model.snapshot import RepoSnapshot
|
|
12
|
+
from repotour.model.tour import Evidence, MissingBasic
|
|
13
|
+
|
|
14
|
+
CI_FILES = frozenset(
|
|
15
|
+
{
|
|
16
|
+
".gitlab-ci.yml",
|
|
17
|
+
".travis.yml",
|
|
18
|
+
"azure-pipelines.yml",
|
|
19
|
+
"bitbucket-pipelines.yml",
|
|
20
|
+
"Jenkinsfile",
|
|
21
|
+
".circleci/config.yml",
|
|
22
|
+
".drone.yml",
|
|
23
|
+
"appveyor.yml",
|
|
24
|
+
}
|
|
25
|
+
)
|
|
26
|
+
CI_DIRS = (".github/workflows/", ".buildkite/", ".woodpecker/")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def find_missing(snapshot: RepoSnapshot) -> tuple[MissingBasic, ...]:
|
|
30
|
+
paths = [f.path for f in snapshot.files]
|
|
31
|
+
checks: list[tuple[str, Callable[[], bool], str, str]] = [
|
|
32
|
+
("README", lambda: any(_at_root(p) and _stem(p) == "readme" for p in paths),
|
|
33
|
+
"README*", "a README at the repository root"),
|
|
34
|
+
("tests", lambda: any(f.category == "test" for f in snapshot.files),
|
|
35
|
+
"tests", "any file classified as a test"),
|
|
36
|
+
("contributing guide", lambda: any(_is_contributing(p) for p in paths),
|
|
37
|
+
"CONTRIBUTING*", "CONTRIBUTING* at the root, in docs/, or in .github/"),
|
|
38
|
+
("license", lambda: any(_at_root(p) and _is_license(p) for p in paths),
|
|
39
|
+
"LICENSE*", "LICENSE*, LICENCE*, or COPYING* at the repository root"),
|
|
40
|
+
("CI configuration", lambda: any(_is_ci(p) for p in paths),
|
|
41
|
+
".github/workflows/", "GitHub Actions, GitLab CI, CircleCI, Jenkins, Azure, "
|
|
42
|
+
"Travis, Bitbucket, Buildkite, Drone, or AppVeyor configuration"),
|
|
43
|
+
] # fmt: skip
|
|
44
|
+
return tuple(
|
|
45
|
+
MissingBasic(
|
|
46
|
+
item=item,
|
|
47
|
+
evidence=(Evidence(kind="file", ref=ref, detail=f"looked for {looked_for}"),),
|
|
48
|
+
)
|
|
49
|
+
for item, present, ref, looked_for in checks
|
|
50
|
+
if not present()
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _at_root(path: str) -> bool:
|
|
55
|
+
return "/" not in path
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _stem(path: str) -> str:
|
|
59
|
+
return repopath.stem(path).lower()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _is_contributing(path: str) -> bool:
|
|
63
|
+
parent = repopath.parent(path)
|
|
64
|
+
return parent in (".", "docs", ".github") and _stem(path) == "contributing"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _is_license(path: str) -> bool:
|
|
68
|
+
name = repopath.name(path).upper()
|
|
69
|
+
return name.startswith(("LICENSE", "LICENCE", "COPYING"))
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _is_ci(path: str) -> bool:
|
|
73
|
+
return path in CI_FILES or path.startswith(CI_DIRS)
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Bulk commits: commits so broad they say nothing about who works on what.
|
|
2
|
+
|
|
3
|
+
A commit is bulk if it touches more files than `thresholds.bulk_commit_files`
|
|
4
|
+
or is listed in `.git-blame-ignore-revs` (as committed at HEAD) or in config
|
|
5
|
+
`ignore_revs`. Ignore lists may use abbreviated SHAs.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from repotour.model.history import CommitRecord
|
|
11
|
+
from repotour.model.snapshot import RepoSnapshot
|
|
12
|
+
|
|
13
|
+
IGNORE_REVS_FILE = ".git-blame-ignore-revs"
|
|
14
|
+
MIN_ABBREVIATED_SHA = 7
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def find_bulk(snapshot: RepoSnapshot, commits: tuple[CommitRecord, ...]) -> dict[str, str]:
|
|
18
|
+
"""Map each bulk commit's SHA to the reason it is bulk."""
|
|
19
|
+
listed = _listed_revs(snapshot)
|
|
20
|
+
limit = snapshot.config.thresholds.bulk_commit_files
|
|
21
|
+
bulk: dict[str, str] = {}
|
|
22
|
+
for commit in commits:
|
|
23
|
+
reason = _listed_reason(commit.sha, listed)
|
|
24
|
+
if reason is None and len(commit.files) > limit:
|
|
25
|
+
reason = f"touches {len(commit.files)} files"
|
|
26
|
+
if reason is not None:
|
|
27
|
+
bulk[commit.sha] = reason
|
|
28
|
+
return bulk
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _listed_revs(snapshot: RepoSnapshot) -> list[tuple[str, str]]:
|
|
32
|
+
revs: list[tuple[str, str]] = []
|
|
33
|
+
if IGNORE_REVS_FILE in snapshot.contents:
|
|
34
|
+
text = snapshot.read_bytes(IGNORE_REVS_FILE).decode("utf-8", "replace")
|
|
35
|
+
for line in text.splitlines():
|
|
36
|
+
rev = line.split("#", 1)[0].strip().lower()
|
|
37
|
+
if len(rev) >= MIN_ABBREVIATED_SHA:
|
|
38
|
+
revs.append((rev, f"listed in {IGNORE_REVS_FILE}"))
|
|
39
|
+
for rev in snapshot.config.ignore_revs:
|
|
40
|
+
cleaned = rev.strip().lower()
|
|
41
|
+
if len(cleaned) >= MIN_ABBREVIATED_SHA:
|
|
42
|
+
revs.append((cleaned, "listed in config ignore_revs"))
|
|
43
|
+
return revs
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _listed_reason(sha: str, listed: list[tuple[str, str]]) -> str | None:
|
|
47
|
+
for rev, reason in listed:
|
|
48
|
+
if sha.startswith(rev):
|
|
49
|
+
return reason
|
|
50
|
+
return None
|
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
"""How to install, run, and test the project, quoted from its own files.
|
|
2
|
+
|
|
3
|
+
Sources come in priority tiers. For each purpose (install, run, test), only
|
|
4
|
+
the highest tier that defines that purpose is used, so a Makefile's `test`
|
|
5
|
+
target hides the CI workflow's test step. Commands are never invented.
|
|
6
|
+
|
|
7
|
+
1. Makefile targets and justfile recipes
|
|
8
|
+
2. package.json scripts
|
|
9
|
+
3. pyproject.toml task runners, tox.ini, noxfile.py
|
|
10
|
+
4. GitHub Actions workflow steps (install and test only)
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
import tomllib
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
import yaml
|
|
22
|
+
|
|
23
|
+
from repotour.model.snapshot import RepoSnapshot
|
|
24
|
+
from repotour.model.tour import Command, Evidence
|
|
25
|
+
|
|
26
|
+
PURPOSES = ("install", "run", "test")
|
|
27
|
+
MAX_PER_PURPOSE = 5
|
|
28
|
+
MAX_OTHER = 8
|
|
29
|
+
SOURCES_LOOKED_AT = (
|
|
30
|
+
"Makefile, justfile, package.json scripts, pyproject.toml task runners, "
|
|
31
|
+
"tox.ini, noxfile.py, and GitHub Actions workflows"
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
_INSTALL_NAMES = frozenset({"install", "setup", "bootstrap", "deps", "dependencies", "init"})
|
|
35
|
+
_RUN_NAMES = frozenset({"run", "start", "serve", "server", "dev", "develop"})
|
|
36
|
+
_TEST_NAME = re.compile(r"^(test|tests|spec|e2e|unit)([-_:].*)?$")
|
|
37
|
+
|
|
38
|
+
_MAKE_TARGET = re.compile(r"^([A-Za-z0-9][\w.-]*)\s*:(?![:=])", re.MULTILINE)
|
|
39
|
+
_JUST_RECIPE = re.compile(r"^@?([A-Za-z0-9][\w-]*)(?:\s+[^:=\n]*)?:(?!=)", re.MULTILINE)
|
|
40
|
+
_NOX_SESSION = re.compile(r"@nox\.session(?:\([^)]*\))?\s*\ndef\s+(\w+)")
|
|
41
|
+
_CI_INSTALL = re.compile(
|
|
42
|
+
r"\b(pip3? install|uv sync|uv pip install|poetry install|pdm install|pipenv install"
|
|
43
|
+
r"|npm (ci|install)|yarn install|pnpm install|bun install|bundle install"
|
|
44
|
+
r"|go mod download|cargo fetch|composer install)\b"
|
|
45
|
+
)
|
|
46
|
+
_CI_TEST = re.compile(
|
|
47
|
+
r"\b(pytest|tox|nox|go test|cargo test|npm (run )?test|yarn test|pnpm test|bun test"
|
|
48
|
+
r"|make test|jest|vitest|mocha|rspec|phpunit|unittest|mvn test|gradle test)\b"
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class _Found:
|
|
54
|
+
purpose: str
|
|
55
|
+
command: str
|
|
56
|
+
ref: str
|
|
57
|
+
detail: str
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def find_commands(snapshot: RepoSnapshot) -> tuple[tuple[Command, ...], list[str]]:
|
|
61
|
+
finder = _Finder(snapshot)
|
|
62
|
+
tiers = [
|
|
63
|
+
finder.makefile() + finder.justfile(),
|
|
64
|
+
finder.package_json(),
|
|
65
|
+
finder.pyproject() + finder.tox() + finder.nox(),
|
|
66
|
+
finder.workflows(),
|
|
67
|
+
]
|
|
68
|
+
chosen: list[_Found] = []
|
|
69
|
+
for purpose in PURPOSES:
|
|
70
|
+
for tier in tiers:
|
|
71
|
+
matching = [f for f in tier if f.purpose == purpose]
|
|
72
|
+
if matching:
|
|
73
|
+
chosen.extend(matching[:MAX_PER_PURPOSE])
|
|
74
|
+
break
|
|
75
|
+
others = [f for tier in tiers for f in tier if f.purpose == "other"]
|
|
76
|
+
chosen.extend(others[:MAX_OTHER])
|
|
77
|
+
commands = tuple(
|
|
78
|
+
Command(
|
|
79
|
+
purpose=f.purpose,
|
|
80
|
+
command=f.command,
|
|
81
|
+
evidence=(Evidence(kind="file", ref=f.ref, detail=f.detail),),
|
|
82
|
+
)
|
|
83
|
+
for f in chosen
|
|
84
|
+
)
|
|
85
|
+
return commands, finder.warnings
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def purpose_of(name: str) -> str:
|
|
89
|
+
lowered = name.lower()
|
|
90
|
+
if lowered in _INSTALL_NAMES:
|
|
91
|
+
return "install"
|
|
92
|
+
if lowered in _RUN_NAMES:
|
|
93
|
+
return "run"
|
|
94
|
+
if _TEST_NAME.match(lowered):
|
|
95
|
+
return "test"
|
|
96
|
+
return "other"
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class _Finder:
|
|
100
|
+
def __init__(self, snapshot: RepoSnapshot) -> None:
|
|
101
|
+
self.snapshot = snapshot
|
|
102
|
+
self.all_paths = {f.path for f in snapshot.files}
|
|
103
|
+
self.warnings: list[str] = []
|
|
104
|
+
|
|
105
|
+
def makefile(self) -> list[_Found]:
|
|
106
|
+
path = next(
|
|
107
|
+
(p for p in ("Makefile", "makefile", "GNUmakefile") if p in self.all_paths), None
|
|
108
|
+
)
|
|
109
|
+
if path is None:
|
|
110
|
+
return []
|
|
111
|
+
text = self._text(path)
|
|
112
|
+
found = []
|
|
113
|
+
for match in _MAKE_TARGET.finditer(text):
|
|
114
|
+
name = match.group(1)
|
|
115
|
+
if name.startswith(".") or "%" in name:
|
|
116
|
+
continue
|
|
117
|
+
recipe = _first_recipe_line(text, match.end())
|
|
118
|
+
detail = f"target {name}: {recipe}" if recipe else f"target {name}"
|
|
119
|
+
found.append(_Found(purpose_of(name), f"make {name}", path, detail))
|
|
120
|
+
return _sorted_unique(found)
|
|
121
|
+
|
|
122
|
+
def justfile(self) -> list[_Found]:
|
|
123
|
+
path = next((p for p in ("justfile", "Justfile", ".justfile") if p in self.all_paths), None)
|
|
124
|
+
if path is None:
|
|
125
|
+
return []
|
|
126
|
+
text = self._text(path)
|
|
127
|
+
found = []
|
|
128
|
+
for match in _JUST_RECIPE.finditer(text):
|
|
129
|
+
name = match.group(1)
|
|
130
|
+
recipe = _first_recipe_line(text, match.end())
|
|
131
|
+
detail = f"recipe {name}: {recipe}" if recipe else f"recipe {name}"
|
|
132
|
+
found.append(_Found(purpose_of(name), f"just {name}", path, detail))
|
|
133
|
+
return _sorted_unique(found)
|
|
134
|
+
|
|
135
|
+
def package_json(self) -> list[_Found]:
|
|
136
|
+
data = self._parse("package.json", json.loads)
|
|
137
|
+
scripts = data.get("scripts") if isinstance(data, dict) else None
|
|
138
|
+
if not isinstance(scripts, dict):
|
|
139
|
+
return []
|
|
140
|
+
manager = self._package_manager()
|
|
141
|
+
found = []
|
|
142
|
+
for name, body in scripts.items():
|
|
143
|
+
if not isinstance(body, str):
|
|
144
|
+
continue
|
|
145
|
+
found.append(
|
|
146
|
+
_Found(
|
|
147
|
+
purpose_of(name),
|
|
148
|
+
_script_command(manager, name),
|
|
149
|
+
"package.json",
|
|
150
|
+
f'script "{name}": {body}',
|
|
151
|
+
)
|
|
152
|
+
)
|
|
153
|
+
return _sorted_unique(found)
|
|
154
|
+
|
|
155
|
+
def pyproject(self) -> list[_Found]:
|
|
156
|
+
data = self._parse("pyproject.toml", tomllib.loads)
|
|
157
|
+
tool = data.get("tool") if isinstance(data, dict) else None
|
|
158
|
+
if not isinstance(tool, dict):
|
|
159
|
+
return []
|
|
160
|
+
found: list[_Found] = []
|
|
161
|
+
runners = (
|
|
162
|
+
("poe", "tasks", "poe"),
|
|
163
|
+
("taskipy", "tasks", "task"),
|
|
164
|
+
("pdm", "scripts", "pdm run"),
|
|
165
|
+
)
|
|
166
|
+
for section, key, prefix in runners:
|
|
167
|
+
tasks = _get(tool, section, key)
|
|
168
|
+
if isinstance(tasks, dict):
|
|
169
|
+
for name in tasks:
|
|
170
|
+
if not name.startswith("_"):
|
|
171
|
+
found.append(
|
|
172
|
+
_Found(
|
|
173
|
+
purpose_of(name),
|
|
174
|
+
f"{prefix} {name}",
|
|
175
|
+
"pyproject.toml",
|
|
176
|
+
f"[tool.{section}.{key}] {name}",
|
|
177
|
+
)
|
|
178
|
+
)
|
|
179
|
+
envs = _get(tool, "hatch", "envs")
|
|
180
|
+
if isinstance(envs, dict):
|
|
181
|
+
for env_name, env in envs.items():
|
|
182
|
+
scripts = env.get("scripts") if isinstance(env, dict) else None
|
|
183
|
+
if not isinstance(scripts, dict):
|
|
184
|
+
continue
|
|
185
|
+
for name in scripts:
|
|
186
|
+
target = name if env_name == "default" else f"{env_name}:{name}"
|
|
187
|
+
found.append(
|
|
188
|
+
_Found(
|
|
189
|
+
purpose_of(name),
|
|
190
|
+
f"hatch run {target}",
|
|
191
|
+
"pyproject.toml",
|
|
192
|
+
f"[tool.hatch.envs.{env_name}.scripts] {name}",
|
|
193
|
+
)
|
|
194
|
+
)
|
|
195
|
+
return _sorted_unique(found)
|
|
196
|
+
|
|
197
|
+
def tox(self) -> list[_Found]:
|
|
198
|
+
if "tox.ini" not in self.all_paths:
|
|
199
|
+
return []
|
|
200
|
+
return [_Found("test", "tox", "tox.ini", "tox configuration")]
|
|
201
|
+
|
|
202
|
+
def nox(self) -> list[_Found]:
|
|
203
|
+
if "noxfile.py" not in self.all_paths:
|
|
204
|
+
return []
|
|
205
|
+
sessions = _NOX_SESSION.findall(self._text("noxfile.py"))
|
|
206
|
+
return _sorted_unique(
|
|
207
|
+
[_Found(purpose_of(s), f"nox -s {s}", "noxfile.py", f"session {s}") for s in sessions]
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
def workflows(self) -> list[_Found]:
|
|
211
|
+
found: list[_Found] = []
|
|
212
|
+
workflow_paths = sorted(
|
|
213
|
+
p
|
|
214
|
+
for p in self.all_paths
|
|
215
|
+
if p.startswith(".github/workflows/") and p.endswith((".yml", ".yaml"))
|
|
216
|
+
)
|
|
217
|
+
for path in workflow_paths:
|
|
218
|
+
data = self._parse(path, yaml.safe_load)
|
|
219
|
+
jobs = data.get("jobs") if isinstance(data, dict) else None
|
|
220
|
+
if not isinstance(jobs, dict):
|
|
221
|
+
continue
|
|
222
|
+
for job_name, job in jobs.items():
|
|
223
|
+
steps = job.get("steps") if isinstance(job, dict) else None
|
|
224
|
+
for step in steps if isinstance(steps, list) else []:
|
|
225
|
+
run = step.get("run") if isinstance(step, dict) else None
|
|
226
|
+
if not isinstance(run, str):
|
|
227
|
+
continue
|
|
228
|
+
for line in (ln.strip() for ln in run.splitlines()):
|
|
229
|
+
purpose = _ci_purpose(line)
|
|
230
|
+
if purpose is not None:
|
|
231
|
+
found.append(_Found(purpose, line, path, f"job {job_name}"))
|
|
232
|
+
unique: dict[tuple[str, str], _Found] = {}
|
|
233
|
+
for f in found:
|
|
234
|
+
unique.setdefault((f.purpose, f.command), f)
|
|
235
|
+
return list(unique.values())
|
|
236
|
+
|
|
237
|
+
def _package_manager(self) -> str:
|
|
238
|
+
for lockfile, manager in (
|
|
239
|
+
("pnpm-lock.yaml", "pnpm"),
|
|
240
|
+
("yarn.lock", "yarn"),
|
|
241
|
+
("bun.lockb", "bun"),
|
|
242
|
+
("bun.lock", "bun"),
|
|
243
|
+
):
|
|
244
|
+
if lockfile in self.all_paths:
|
|
245
|
+
return manager
|
|
246
|
+
return "npm"
|
|
247
|
+
|
|
248
|
+
def _text(self, path: str) -> str:
|
|
249
|
+
return self.snapshot.read_bytes(path).decode("utf-8", "replace")
|
|
250
|
+
|
|
251
|
+
def _parse(self, path: str, loader: Any) -> Any:
|
|
252
|
+
if path not in self.all_paths:
|
|
253
|
+
return None
|
|
254
|
+
try:
|
|
255
|
+
return loader(self._text(path))
|
|
256
|
+
except (ValueError, yaml.YAMLError) as exc:
|
|
257
|
+
# Manifests are reported once, by project detection; only report others.
|
|
258
|
+
if path not in ("package.json", "pyproject.toml"):
|
|
259
|
+
self.warnings.append(f"{path}: could not be parsed ({exc})")
|
|
260
|
+
return None
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _script_command(manager: str, name: str) -> str:
|
|
264
|
+
if name in ("test", "start") and manager != "bun":
|
|
265
|
+
return f"{manager} {name}"
|
|
266
|
+
return f"{manager} run {name}"
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _first_recipe_line(text: str, offset: int) -> str | None:
|
|
270
|
+
for line in text[offset:].splitlines()[1:]:
|
|
271
|
+
if not line.strip():
|
|
272
|
+
continue
|
|
273
|
+
if line.startswith(("\t", " ")):
|
|
274
|
+
text = line.strip().lstrip("@")
|
|
275
|
+
return f"{text[:-1].rstrip()} …" if text.endswith("\\") else text
|
|
276
|
+
return None
|
|
277
|
+
return None
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _ci_purpose(line: str) -> str | None:
|
|
281
|
+
if not line or line.startswith(("#", "echo ", "cd ")) or "${{" in line:
|
|
282
|
+
return None
|
|
283
|
+
if _CI_INSTALL.search(line):
|
|
284
|
+
return "install"
|
|
285
|
+
if _CI_TEST.search(line):
|
|
286
|
+
return "test"
|
|
287
|
+
return None
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _sorted_unique(found: list[_Found]) -> list[_Found]:
|
|
291
|
+
unique = {f.command: f for f in found}
|
|
292
|
+
return sorted(unique.values(), key=lambda f: f.command)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _get(table: dict[str, Any], *keys: str) -> Any:
|
|
296
|
+
value: Any = table
|
|
297
|
+
for key in keys:
|
|
298
|
+
if not isinstance(value, dict):
|
|
299
|
+
return None
|
|
300
|
+
value = value.get(key)
|
|
301
|
+
return value
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""Dependency graph: which modules import which, inside the repository.
|
|
2
|
+
|
|
3
|
+
Only static imports count. Test, generated, and vendored files are left out.
|
|
4
|
+
Imports that resolve outside the repository are counted, not drawn. A file
|
|
5
|
+
that fails to parse becomes a warning and is skipped.
|
|
6
|
+
|
|
7
|
+
PageRank (over importer -> imported edges) names the core modules; modules
|
|
8
|
+
nothing imports that are not entry points (and not empty) are leaf modules. Graphs over
|
|
9
|
+
`thresholds.graph_max_nodes` collapse to directories unless `full_graph`.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import math
|
|
15
|
+
from collections import Counter
|
|
16
|
+
from collections.abc import Callable
|
|
17
|
+
|
|
18
|
+
from repotour.analyzers.base import AnalyzerResult
|
|
19
|
+
from repotour.graph.collapse import ROOT_GROUP, collapse
|
|
20
|
+
from repotour.graph.pagerank import pagerank
|
|
21
|
+
from repotour.languages import ADAPTERS
|
|
22
|
+
from repotour.languages.base import LanguageAdapter, RawImport
|
|
23
|
+
from repotour.model.serde import to_data
|
|
24
|
+
from repotour.model.snapshot import RepoSnapshot
|
|
25
|
+
from repotour.model.tour import (
|
|
26
|
+
CoreModule,
|
|
27
|
+
DependencyGraphSection,
|
|
28
|
+
Evidence,
|
|
29
|
+
ExternalImports,
|
|
30
|
+
GraphNode,
|
|
31
|
+
ImportEdge,
|
|
32
|
+
LeafModule,
|
|
33
|
+
)
|
|
34
|
+
from repotour.scan.filetypes import is_peripheral
|
|
35
|
+
|
|
36
|
+
EXTERNAL_EXAMPLES = 5
|
|
37
|
+
PARTIAL_EXAMPLES = 5
|
|
38
|
+
MAX_LEAVES = 10
|
|
39
|
+
GRAPH_CATEGORIES = ("source",)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class DependenciesAnalyzer:
|
|
43
|
+
name = "dependencies"
|
|
44
|
+
version = 6
|
|
45
|
+
depends_on: tuple[str, ...] = ("entry_points",)
|
|
46
|
+
|
|
47
|
+
def run(self, snapshot: RepoSnapshot, inputs: dict[str, AnalyzerResult]) -> AnalyzerResult:
|
|
48
|
+
all_paths = [f.path for f in snapshot.files]
|
|
49
|
+
candidates = [f.path for f in snapshot.files if f.category in GRAPH_CATEGORIES]
|
|
50
|
+
warnings: list[str] = []
|
|
51
|
+
languages: list[str] = []
|
|
52
|
+
nodes: list[str] = []
|
|
53
|
+
lines: dict[tuple[str, str], int] = {} # (source, target) -> first import line
|
|
54
|
+
external: dict[str, tuple[str, RawImport]] = {} # name -> first (path, import)
|
|
55
|
+
reached: set[str] = set() # files reached by some import (whole Go packages)
|
|
56
|
+
|
|
57
|
+
for adapter in ADAPTERS:
|
|
58
|
+
paths = sorted(p for p in candidates if adapter.handles(p))
|
|
59
|
+
if not paths:
|
|
60
|
+
continue
|
|
61
|
+
languages.append(adapter.name)
|
|
62
|
+
parsed = _parse_all(adapter, paths, snapshot, warnings)
|
|
63
|
+
index = adapter.build_index(list(parsed), all_paths, _reader(snapshot))
|
|
64
|
+
nodes.extend(parsed)
|
|
65
|
+
for path, imports in parsed.items():
|
|
66
|
+
for raw in imports:
|
|
67
|
+
targets = adapter.resolve(raw, path, index)
|
|
68
|
+
if not targets and not raw.module.startswith((".", "/")):
|
|
69
|
+
external.setdefault(adapter.external_name(raw), (path, raw))
|
|
70
|
+
for target in targets:
|
|
71
|
+
if target != path:
|
|
72
|
+
lines.setdefault((path, target), raw.line)
|
|
73
|
+
reached.update(adapter.imported_together(target, index))
|
|
74
|
+
|
|
75
|
+
nodes.sort()
|
|
76
|
+
edges = sorted(lines)
|
|
77
|
+
ranks = pagerank(nodes, edges)
|
|
78
|
+
entry_paths = _entry_paths(inputs)
|
|
79
|
+
config = snapshot.config
|
|
80
|
+
limit = len(nodes) if config.full_graph else config.thresholds.graph_max_nodes
|
|
81
|
+
drawn = collapse(nodes, edges, ranks, max(limit, 1))
|
|
82
|
+
empty = {f.path for f in snapshot.files if f.lines == 0}
|
|
83
|
+
leaves = _leaves([n for n in nodes if n not in empty | reached], edges, entry_paths)
|
|
84
|
+
|
|
85
|
+
section = DependencyGraphSection(
|
|
86
|
+
languages=tuple(sorted(languages)),
|
|
87
|
+
modules=len(nodes),
|
|
88
|
+
imports=len(edges),
|
|
89
|
+
depth=drawn.depth,
|
|
90
|
+
nodes=tuple(GraphNode(id=g, modules=len(drawn.members[g])) for g in drawn.nodes),
|
|
91
|
+
edges=tuple(
|
|
92
|
+
_edge(source, target, behind, lines)
|
|
93
|
+
for (source, target), behind in sorted(drawn.edges.items())
|
|
94
|
+
),
|
|
95
|
+
omitted=drawn.omitted,
|
|
96
|
+
external=_external_claim(external),
|
|
97
|
+
core=_core(nodes, edges, ranks, config.thresholds.core_module_share),
|
|
98
|
+
leaves=tuple(leaves[:MAX_LEAVES]),
|
|
99
|
+
leaves_omitted=max(0, len(leaves) - MAX_LEAVES),
|
|
100
|
+
)
|
|
101
|
+
data = to_data(section)
|
|
102
|
+
# Module-level graph for the reading order (the drawn graph may be collapsed).
|
|
103
|
+
data["module_edges"] = [[s, t, lines[(s, t)]] for s, t in edges]
|
|
104
|
+
data["ranks"] = ranks
|
|
105
|
+
return AnalyzerResult(analyzer=self.name, data=data, warnings=warnings)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _reader(snapshot: RepoSnapshot) -> Callable[[str], str]:
|
|
109
|
+
def read(path: str) -> str:
|
|
110
|
+
return snapshot.read_bytes(path).decode("utf-8", "replace")
|
|
111
|
+
|
|
112
|
+
return read
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _parse_all(
|
|
116
|
+
adapter: LanguageAdapter, paths: list[str], snapshot: RepoSnapshot, warnings: list[str]
|
|
117
|
+
) -> dict[str, tuple[RawImport, ...]]:
|
|
118
|
+
"""Imports per file. tree-sitter recovers from syntax it does not know, so
|
|
119
|
+
a partially parsed file keeps the imports it yielded; all such files are
|
|
120
|
+
reported in one warning rather than one each."""
|
|
121
|
+
parsed: dict[str, tuple[RawImport, ...]] = {}
|
|
122
|
+
partial: list[str] = []
|
|
123
|
+
for path in paths:
|
|
124
|
+
result = snapshot.parse(adapter, path)
|
|
125
|
+
if result.has_error:
|
|
126
|
+
partial.append(path)
|
|
127
|
+
parsed[path] = result.imports
|
|
128
|
+
if partial:
|
|
129
|
+
examples = ", ".join(partial[:PARTIAL_EXAMPLES])
|
|
130
|
+
more = (
|
|
131
|
+
f", and {len(partial) - PARTIAL_EXAMPLES} more"
|
|
132
|
+
if len(partial) > PARTIAL_EXAMPLES
|
|
133
|
+
else ""
|
|
134
|
+
)
|
|
135
|
+
warnings.append(
|
|
136
|
+
f"{len(partial)} {adapter.name} file{'' if len(partial) == 1 else 's'} had syntax "
|
|
137
|
+
f"the parser could not fully read; their imports were still used: {examples}{more}"
|
|
138
|
+
)
|
|
139
|
+
return parsed
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _entry_paths(inputs: dict[str, AnalyzerResult]) -> set[str]:
|
|
143
|
+
result = inputs.get("entry_points")
|
|
144
|
+
entries = result.data.get("entries", []) if result is not None else []
|
|
145
|
+
return {e["path"] for e in entries if isinstance(e, dict)}
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _edge(
|
|
149
|
+
source: str, target: str, behind: list[tuple[str, str]], lines: dict[tuple[str, str], int]
|
|
150
|
+
) -> ImportEdge:
|
|
151
|
+
first = behind[0]
|
|
152
|
+
detail = f"line {lines[first]}"
|
|
153
|
+
if len(behind) > 1 or (source, target) != first:
|
|
154
|
+
detail = f"{len(behind)} module import{'' if len(behind) == 1 else 's'}, e.g. {detail}"
|
|
155
|
+
return ImportEdge(
|
|
156
|
+
source=source,
|
|
157
|
+
target=target,
|
|
158
|
+
count=len(behind),
|
|
159
|
+
evidence=(Evidence(kind="import", ref=f"{first[0]} -> {first[1]}", detail=detail),),
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _core(
|
|
164
|
+
nodes: list[str], edges: list[tuple[str, str]], ranks: dict[str, float], share: float
|
|
165
|
+
) -> tuple[CoreModule, ...]:
|
|
166
|
+
importers: dict[str, list[str]] = {n: [] for n in nodes}
|
|
167
|
+
for source, target in edges:
|
|
168
|
+
importers[target].append(source)
|
|
169
|
+
ordered = sorted(nodes, key=lambda n: (-ranks[n], n))
|
|
170
|
+
position = {n: i + 1 for i, n in enumerate(ordered)}
|
|
171
|
+
imported = [n for n in ordered if importers[n] and not is_peripheral(n)]
|
|
172
|
+
count = min(len(imported), max(1, math.ceil(len(nodes) * share))) if imported else 0
|
|
173
|
+
return tuple(
|
|
174
|
+
CoreModule(
|
|
175
|
+
path=n,
|
|
176
|
+
importers=len(importers[n]),
|
|
177
|
+
rank=position[n],
|
|
178
|
+
evidence=(
|
|
179
|
+
Evidence(
|
|
180
|
+
kind="import",
|
|
181
|
+
ref=f"{importers[n][0]} -> {n}",
|
|
182
|
+
detail=f"imported by {len(importers[n])} module"
|
|
183
|
+
f"{'' if len(importers[n]) == 1 else 's'}; "
|
|
184
|
+
f"PageRank {position[n]} of {len(nodes)}",
|
|
185
|
+
),
|
|
186
|
+
),
|
|
187
|
+
)
|
|
188
|
+
for n in imported[:count]
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _leaves(
|
|
193
|
+
nodes: list[str], edges: list[tuple[str, str]], entry_paths: set[str]
|
|
194
|
+
) -> list[LeafModule]:
|
|
195
|
+
in_degree = Counter(target for _, target in edges)
|
|
196
|
+
return [
|
|
197
|
+
LeafModule(
|
|
198
|
+
path=n,
|
|
199
|
+
evidence=(
|
|
200
|
+
Evidence(kind="file", ref=n, detail="no module imports it; not an entry point"),
|
|
201
|
+
),
|
|
202
|
+
)
|
|
203
|
+
for n in nodes
|
|
204
|
+
if in_degree[n] == 0 and n not in entry_paths and not is_peripheral(n)
|
|
205
|
+
]
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _external_claim(external: dict[str, tuple[str, RawImport]]) -> ExternalImports | None:
|
|
209
|
+
if not external:
|
|
210
|
+
return None
|
|
211
|
+
names = sorted(external)
|
|
212
|
+
return ExternalImports(
|
|
213
|
+
count=len(names),
|
|
214
|
+
examples=tuple(names[:EXTERNAL_EXAMPLES]),
|
|
215
|
+
evidence=tuple(
|
|
216
|
+
Evidence(
|
|
217
|
+
kind="import",
|
|
218
|
+
ref=f"{external[name][0]} -> {name}",
|
|
219
|
+
detail=f"line {external[name][1].line}",
|
|
220
|
+
)
|
|
221
|
+
for name in names[:3]
|
|
222
|
+
),
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
__all__ = ["ROOT_GROUP", "DependenciesAnalyzer"]
|