leyline-code 0.8.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- leyline/__init__.py +3 -0
- leyline/adapters/__init__.py +19 -0
- leyline/adapters/csharp.py +1026 -0
- leyline/adapters/generic.py +844 -0
- leyline/adapters/python.py +929 -0
- leyline/adapters/typescript.py +1512 -0
- leyline/affected.py +509 -0
- leyline/agent_skills.py +311 -0
- leyline/change.py +500 -0
- leyline/channels.py +1782 -0
- leyline/cli.py +1029 -0
- leyline/cluster.py +149 -0
- leyline/context.py +531 -0
- leyline/coupling.py +485 -0
- leyline/coverage.py +376 -0
- leyline/diagrams.py +680 -0
- leyline/diff.py +1078 -0
- leyline/drift.py +747 -0
- leyline/exact.py +410 -0
- leyline/explain.py +973 -0
- leyline/export.py +673 -0
- leyline/fncoupling.py +328 -0
- leyline/formats.py +923 -0
- leyline/grade.py +207 -0
- leyline/incremental.py +931 -0
- leyline/indexer.py +3733 -0
- leyline/learnings.py +743 -0
- leyline/loop.py +584 -0
- leyline/model.py +142 -0
- leyline/outline.py +1066 -0
- leyline/packing.py +86 -0
- leyline/patterns.py +478 -0
- leyline/pr.py +1249 -0
- leyline/props.py +255 -0
- leyline/query.py +683 -0
- leyline/quick.py +853 -0
- leyline/related.py +266 -0
- leyline/removal.py +231 -0
- leyline/renames.py +231 -0
- leyline/rereview.py +375 -0
- leyline/roslyn_refs/Program.cs +179 -0
- leyline/roslyn_refs/RoslynRefs.csproj +16 -0
- leyline/rules.py +171 -0
- leyline/schema.sql +178 -0
- leyline/scip_pb2.py +97 -0
- leyline/server.py +1230 -0
- leyline/skills/leyline-adversarial-review/SKILL.md +121 -0
- leyline/skills/leyline-ask/SKILL.md +57 -0
- leyline/skills/leyline-change-impact/SKILL.md +79 -0
- leyline/skills/leyline-explain-flow/SKILL.md +70 -0
- leyline/skills/leyline-explore-module/SKILL.md +55 -0
- leyline/skills/leyline-pr-review/SKILL.md +77 -0
- leyline/skills/leyline-quick-change/SKILL.md +63 -0
- leyline/skills/leyline-spec/SKILL.md +179 -0
- leyline/skills/leyline-tour/SKILL.md +46 -0
- leyline/spec.py +1819 -0
- leyline/store.py +560 -0
- leyline/tours.py +352 -0
- leyline/verdicts.py +390 -0
- leyline/viewer/viewer.html +1622 -0
- leyline_code-0.8.1.dist-info/METADATA +1325 -0
- leyline_code-0.8.1.dist-info/RECORD +65 -0
- leyline_code-0.8.1.dist-info/WHEEL +4 -0
- leyline_code-0.8.1.dist-info/entry_points.txt +2 -0
- leyline_code-0.8.1.dist-info/licenses/LICENSE +674 -0
leyline/coupling.py
ADDED
|
@@ -0,0 +1,485 @@
|
|
|
1
|
+
"""Change coupling from git history: files that usually change in the same commits.
|
|
2
|
+
|
|
3
|
+
The map links code that calls, imports or crosses a channel to other code. It cannot see a doc that has to be updated
|
|
4
|
+
with a rule, a JSON schema that has to agree with its reader, a fixture a test compares with, or the other side of a
|
|
5
|
+
protocol written in another language with no shared name. Git history can: if `schema/x.json` changed in 7 of the 9
|
|
6
|
+
commits that changed `validate.py`, a change to `validate.py` that leaves `schema/x.json` alone is worth a question.
|
|
7
|
+
|
|
8
|
+
The measure is CodeScene's. Over the commits read, for files A and B:
|
|
9
|
+
- changes(A): commits that changed A
|
|
10
|
+
- together(A, B): commits that changed both
|
|
11
|
+
- confidence(A -> B) = together / changes(A): how often a change to A came with a change to B
|
|
12
|
+
- support = together / commits read
|
|
13
|
+
|
|
14
|
+
The same is counted from a file to a folder (a commit that changed A and any file in the folder), because some
|
|
15
|
+
partners are a new file each time: a rule that comes with a new test case in `cases/<name>/` never couples to any one
|
|
16
|
+
case, but it couples to `cases/`. A folder is reported only when no deeper folder says the same, no single file in
|
|
17
|
+
it is reported for most of the same commits, and it is not at the top of the repository (`tests/`, `docs/`), which is
|
|
18
|
+
too broad to act on.
|
|
19
|
+
|
|
20
|
+
Which commits are read: at most MAX_COMMITS, from the two years before the commit read from, merges left out. A
|
|
21
|
+
commit that changed more than BULK files (a formatter run, a licence header, a vendored import) is left out: it says
|
|
22
|
+
those files were touched at once, not that they belong together. Renames are followed, so a file's older commits
|
|
23
|
+
count under its name now; files that no longer exist are not reported.
|
|
24
|
+
|
|
25
|
+
File level here. Function-level coupling maps each commit's diff hunks to the functions they fall in, and a hunk's
|
|
26
|
+
line numbers are those of its own commit, so it means parsing old versions of files: too slow for a whole history,
|
|
27
|
+
so leyline.fncoupling does it only for the few functions a plan names or a pull request edits (for_spec, for_pr).
|
|
28
|
+
|
|
29
|
+
The result is kept in the store (coupling_runs, coupling_files, coupling_pairs, coupling_dirs) under the commit it
|
|
30
|
+
was read from, so it is worked out once per commit.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import itertools
|
|
36
|
+
import re
|
|
37
|
+
import subprocess
|
|
38
|
+
import time
|
|
39
|
+
from collections import Counter
|
|
40
|
+
from pathlib import Path
|
|
41
|
+
from typing import Iterable, Optional
|
|
42
|
+
|
|
43
|
+
from . import store
|
|
44
|
+
|
|
45
|
+
MAX_COMMITS = 1000
|
|
46
|
+
YEARS = 2
|
|
47
|
+
BULK = 50 # a commit that changed more files than this is a bulk change, left out
|
|
48
|
+
MIN_TOGETHER = 3 # report a pair only when it changed together at least this often
|
|
49
|
+
MIN_CONFIDENCE = 0.5 # and at least this share of the first file's commits changed the second
|
|
50
|
+
KEEP = 2 # pairs are stored from this many shared commits up, so a lower threshold can still be asked
|
|
51
|
+
KEEP_RUNS = 3 # results kept per repository (HEAD, and the bases of pull requests reviewed)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _git(root: Path, *args: str, timeout: float = 300) -> Optional[bytes]:
|
|
55
|
+
try:
|
|
56
|
+
out = subprocess.run(["git", "-C", str(root), *args], capture_output=True, timeout=timeout, stdin=subprocess.DEVNULL)
|
|
57
|
+
except (OSError, subprocess.SubprocessError):
|
|
58
|
+
return None
|
|
59
|
+
return out.stdout if out.returncode == 0 else None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def head(root: Path) -> Optional[str]:
|
|
63
|
+
out = _git(root, "rev-parse", "--verify", "-q", "HEAD^{commit}")
|
|
64
|
+
return out.decode().strip() if out else None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _commits(root: Path, sha: str, max_commits: int, years: float) -> Optional[list[list[tuple[str, ...]]]]:
|
|
68
|
+
"""Each commit's changes, newest first: (status, path) or, for a rename or copy, (status, old path, new path)."""
|
|
69
|
+
when = _git(root, "log", "-1", "--format=%ct", sha)
|
|
70
|
+
if when is None:
|
|
71
|
+
return None
|
|
72
|
+
since = int(when.decode().strip() or 0) - int(years * 365.25 * 86400)
|
|
73
|
+
out = _git(root, "-c", "core.quotePath=false", "log", "--no-merges", "-M", "--name-status", "-z", "--format=%x01%H",
|
|
74
|
+
f"-n{max_commits}", f"--since={since}", sha, "--", ".") # . : only the folder mapped
|
|
75
|
+
if out is None:
|
|
76
|
+
return None
|
|
77
|
+
commits = []
|
|
78
|
+
for chunk in out.split(b"\x01")[1:]:
|
|
79
|
+
tokens = chunk.split(b"\0")[1:] # the first is the commit's sha
|
|
80
|
+
changes, k = [], 0
|
|
81
|
+
while k < len(tokens):
|
|
82
|
+
status = tokens[k].decode("utf-8", "replace").strip()
|
|
83
|
+
if not status:
|
|
84
|
+
k += 1
|
|
85
|
+
continue
|
|
86
|
+
width = 3 if status[:1] in ("R", "C") else 2
|
|
87
|
+
paths = [t.decode("utf-8", "replace") for t in tokens[k + 1:k + width]]
|
|
88
|
+
if len(paths) == width - 1:
|
|
89
|
+
changes.append((status[:1], *paths))
|
|
90
|
+
k += width
|
|
91
|
+
commits.append(changes)
|
|
92
|
+
return commits
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _folders(path: str) -> list[str]:
|
|
96
|
+
parts = path.split("/")[:-1]
|
|
97
|
+
return ["/".join(parts[:k]) + "/" for k in range(1, len(parts) + 1)]
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def compute(root: Path, sha: str, max_commits: int = MAX_COMMITS, years: float = YEARS, bulk: int = BULK) -> Optional[dict]:
|
|
101
|
+
"""Read the history behind `sha` and count, for each file, each pair of files (a < b) and each file and folder
|
|
102
|
+
outside it, the commits that changed them. None outside git."""
|
|
103
|
+
commits = _commits(root, sha, max_commits, years)
|
|
104
|
+
if commits is None:
|
|
105
|
+
return None
|
|
106
|
+
# A repository mapped from a folder inside a larger git repository: git names files from the top, the map from
|
|
107
|
+
# the folder. Only the folder's files count, under the map's names.
|
|
108
|
+
prefix = (_git(root, "rev-parse", "--show-prefix") or b"").decode("utf-8", "replace").strip()
|
|
109
|
+
listed = _git(root, "ls-tree", "-r", "-z", "--full-tree", "--name-only", sha)
|
|
110
|
+
alive = {p.decode("utf-8", "replace") for p in (listed or b"").split(b"\0") if p}
|
|
111
|
+
alive = {p[len(prefix):] for p in alive if p.startswith(prefix)}
|
|
112
|
+
now: dict[str, str] = {} # an older name -> the name the file has at `sha`
|
|
113
|
+
changes: Counter = Counter()
|
|
114
|
+
pairs: Counter = Counter()
|
|
115
|
+
dirs: Counter = Counter()
|
|
116
|
+
read = skipped = 0
|
|
117
|
+
for c in commits: # newest first, so a rename is seen before the commits that used the old name
|
|
118
|
+
files = set()
|
|
119
|
+
for ch in c:
|
|
120
|
+
if ch[0] in ("R", "C"):
|
|
121
|
+
old, new = ch[1], ch[2]
|
|
122
|
+
cur = now.get(new, new)
|
|
123
|
+
if ch[0] == "R":
|
|
124
|
+
now[old] = cur
|
|
125
|
+
files.add(cur)
|
|
126
|
+
else:
|
|
127
|
+
files.add(now.get(ch[1], ch[1]))
|
|
128
|
+
if not files:
|
|
129
|
+
continue
|
|
130
|
+
if len(files) > bulk:
|
|
131
|
+
skipped += 1
|
|
132
|
+
continue
|
|
133
|
+
files = sorted(f[len(prefix):] for f in files if f.startswith(prefix) and f[len(prefix):] in alive
|
|
134
|
+
and not f[len(prefix):].startswith(".leyline/"))
|
|
135
|
+
if not files:
|
|
136
|
+
continue
|
|
137
|
+
read += 1
|
|
138
|
+
changes.update(files)
|
|
139
|
+
pairs.update(itertools.combinations(files, 2))
|
|
140
|
+
touched = {d for f in files for d in _folders(f)}
|
|
141
|
+
for f in files:
|
|
142
|
+
own = set(_folders(f))
|
|
143
|
+
dirs.update((f, d) for d in touched if d not in own)
|
|
144
|
+
return {"commits": read, "bulk": skipped, "changes": dict(changes), "pairs": dict(pairs), "dirs": dict(dirs)}
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
TABLES = ("coupling_runs", "coupling_files", "coupling_pairs", "coupling_dirs")
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def ensure(con, repo: str, root: Path, sha: Optional[str] = None) -> Optional[dict]:
|
|
151
|
+
"""The coupling run for a repository at a commit (HEAD by default), worked out now if the store does not hold it.
|
|
152
|
+
None when the repository is not under git or has no commits."""
|
|
153
|
+
root = Path(root)
|
|
154
|
+
sha = sha or head(root)
|
|
155
|
+
if not sha:
|
|
156
|
+
return None
|
|
157
|
+
row = con.execute("SELECT commits, bulk, seconds FROM coupling_runs WHERE repo_id = ? AND sha = ?", (repo, sha)).fetchone()
|
|
158
|
+
if row is not None:
|
|
159
|
+
return {"repo": repo, "sha": sha, "commits": row[0], "bulk": row[1], "seconds": row[2], "cached": True}
|
|
160
|
+
t0 = time.perf_counter()
|
|
161
|
+
r = compute(root, sha)
|
|
162
|
+
if r is None:
|
|
163
|
+
return None
|
|
164
|
+
seconds = round(time.perf_counter() - t0, 2)
|
|
165
|
+
with con:
|
|
166
|
+
old = [x[0] for x in con.execute("SELECT sha FROM coupling_runs WHERE repo_id = ? ORDER BY created DESC", (repo,))][KEEP_RUNS - 1:]
|
|
167
|
+
for s in old + [sha]:
|
|
168
|
+
for table in TABLES:
|
|
169
|
+
con.execute(f"DELETE FROM {table} WHERE repo_id = ? AND sha = ?", (repo, s))
|
|
170
|
+
con.execute("INSERT INTO coupling_runs VALUES (?,?,?,?,?,?)", (repo, sha, r["commits"], r["bulk"], seconds, time.time()))
|
|
171
|
+
con.executemany("INSERT INTO coupling_files VALUES (?,?,?,?)", ((repo, sha, p, n) for p, n in r["changes"].items()))
|
|
172
|
+
con.executemany("INSERT INTO coupling_pairs VALUES (?,?,?,?,?)",
|
|
173
|
+
((repo, sha, a, b, n) for (a, b), n in r["pairs"].items() if n >= KEEP))
|
|
174
|
+
con.executemany("INSERT INTO coupling_dirs VALUES (?,?,?,?,?)",
|
|
175
|
+
((repo, sha, f, d, n) for (f, d), n in r["dirs"].items() if n >= KEEP))
|
|
176
|
+
return {"repo": repo, "sha": sha, "commits": r["commits"], "bulk": r["bulk"], "seconds": seconds, "cached": False}
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def partners(con, run: dict, paths: Iterable[str], min_together: int = MIN_TOGETHER,
|
|
180
|
+
min_confidence: float = MIN_CONFIDENCE, folders: bool = False) -> dict[str, list[dict]]:
|
|
181
|
+
"""path -> the files that usually change with it, strongest first: {path, together, changes (of the first file),
|
|
182
|
+
confidence, partner_changes, support}. With `folders`, also folders whose files often change with it and that no
|
|
183
|
+
deeper folder or single file in them explains ({..., "folder": True}, path ending in /)."""
|
|
184
|
+
repo, sha, total = run["repo"], run["sha"], max(run["commits"], 1)
|
|
185
|
+
out = {}
|
|
186
|
+
for p in dict.fromkeys(paths):
|
|
187
|
+
row = con.execute("SELECT changes FROM coupling_files WHERE repo_id = ? AND sha = ? AND path = ?", (repo, sha, p)).fetchone()
|
|
188
|
+
if row is None or row[0] < min_together:
|
|
189
|
+
out[p] = []
|
|
190
|
+
continue
|
|
191
|
+
n = row[0]
|
|
192
|
+
rows = con.execute(
|
|
193
|
+
"SELECT CASE WHEN c.a = ? THEN c.b ELSE c.a END AS other, c.together, f.changes FROM coupling_pairs c"
|
|
194
|
+
" JOIN coupling_files f ON f.repo_id = c.repo_id AND f.sha = c.sha AND f.path = (CASE WHEN c.a = ? THEN c.b ELSE c.a END)"
|
|
195
|
+
" WHERE c.repo_id = ? AND c.sha = ? AND (c.a = ? OR c.b = ?) AND c.together >= ?",
|
|
196
|
+
(p, p, repo, sha, p, p, min_together)).fetchall()
|
|
197
|
+
found = [{"path": r[0], "together": r[1], "changes": n, "confidence": round(r[1] / n, 2), "partner_changes": r[2],
|
|
198
|
+
"support": round(r[1] / total, 3)} for r in rows if r[1] / n >= min_confidence]
|
|
199
|
+
if folders:
|
|
200
|
+
ds = {r[0]: r[1] for r in con.execute("SELECT dir, together FROM coupling_dirs WHERE repo_id = ? AND sha = ?"
|
|
201
|
+
" AND path = ? AND together >= ?", (repo, sha, p, min_together))
|
|
202
|
+
if r[1] / n >= min_confidence}
|
|
203
|
+
for d, k in ds.items():
|
|
204
|
+
if d.count("/") < 2: # a folder at the top (tests/, docs/) is too broad to act on
|
|
205
|
+
continue
|
|
206
|
+
deeper = any(o != d and o.startswith(d) for o in ds)
|
|
207
|
+
# one file in it already reported for most of the folder's commits: the folder adds nothing
|
|
208
|
+
one_file = any(x["path"].startswith(d) and x["together"] >= 0.75 * k for x in found)
|
|
209
|
+
if not deeper and not one_file:
|
|
210
|
+
found.append({"path": d, "together": k, "changes": n, "confidence": round(k / n, 2), "folder": True,
|
|
211
|
+
"support": round(k / total, 3)})
|
|
212
|
+
out[p] = sorted(found, key=lambda x: (-x["confidence"], -x["together"], x["path"]))
|
|
213
|
+
return out
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def strongest(con, run: dict, min_together: int = MIN_TOGETHER, min_confidence: float = MIN_CONFIDENCE, limit: int = 20) -> list[dict]:
|
|
217
|
+
"""The most coupled pairs of files in a repository, each in its stronger direction (from the file that changed
|
|
218
|
+
less often, so the share is of its commits)."""
|
|
219
|
+
repo, sha, total = run["repo"], run["sha"], max(run["commits"], 1)
|
|
220
|
+
rows = con.execute(
|
|
221
|
+
"SELECT c.a, c.b, c.together, fa.changes, fb.changes FROM coupling_pairs c"
|
|
222
|
+
" JOIN coupling_files fa ON fa.repo_id = c.repo_id AND fa.sha = c.sha AND fa.path = c.a"
|
|
223
|
+
" JOIN coupling_files fb ON fb.repo_id = c.repo_id AND fb.sha = c.sha AND fb.path = c.b"
|
|
224
|
+
" WHERE c.repo_id = ? AND c.sha = ? AND c.together >= ?", (repo, sha, min_together)).fetchall()
|
|
225
|
+
out = []
|
|
226
|
+
for a, b, n, ca, cb in rows:
|
|
227
|
+
first, other, cf, co = (a, b, ca, cb) if ca <= cb else (b, a, cb, ca)
|
|
228
|
+
if n / cf >= min_confidence:
|
|
229
|
+
out.append({"file": first, "path": other, "together": n, "changes": cf, "confidence": round(n / cf, 2),
|
|
230
|
+
"partner_changes": co, "support": round(n / total, 3)})
|
|
231
|
+
out.sort(key=lambda x: (-x["together"], -x["confidence"], x["file"], x["path"]))
|
|
232
|
+
return out[:limit]
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def missed(con, run: dict, touched: Iterable[str], named, limit: int = 0, **thresholds) -> list[dict]:
|
|
236
|
+
"""Files (and folders) that usually change with one of `touched` and that `named(path)` says nothing covers,
|
|
237
|
+
strongest first, each once, with the touched file it most often changed with. `named` is a set of paths or a
|
|
238
|
+
function; a folder is covered when anything touched or named is in it."""
|
|
239
|
+
is_named = named if callable(named) else (lambda p, s=set(named): p in s)
|
|
240
|
+
touched = list(dict.fromkeys(touched))
|
|
241
|
+
best: dict[str, dict] = {}
|
|
242
|
+
also: dict[str, list] = {}
|
|
243
|
+
for f, ps in partners(con, run, touched, folders=True, **thresholds).items():
|
|
244
|
+
for x in ps:
|
|
245
|
+
p = x["path"]
|
|
246
|
+
if p in touched or is_named(p):
|
|
247
|
+
continue
|
|
248
|
+
if x.get("folder") and any(t.startswith(p) for t in touched):
|
|
249
|
+
continue
|
|
250
|
+
item = {"file": f, **x}
|
|
251
|
+
also.setdefault(p, []).append(f)
|
|
252
|
+
if p not in best or (x["confidence"], x["together"]) > (best[p]["confidence"], best[p]["together"]):
|
|
253
|
+
best[p] = item
|
|
254
|
+
out = [{**x, "also_with": [f for f in also[p] if f != x["file"]]} for p, x in best.items()
|
|
255
|
+
if not (x.get("folder") and any(o != p and o.startswith(p) for o in best))] # said by what is in it
|
|
256
|
+
out.sort(key=lambda x: (-x["confidence"], -x["together"], bool(x.get("folder")), x["path"]))
|
|
257
|
+
return out[:limit] if limit else out
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def line(x: dict, tail: str = "") -> str:
|
|
261
|
+
"""`B` changed in 7 of the 9 commits that changed `A`; <tail>."""
|
|
262
|
+
also = f" (and often with {_list(x['also_with'], 2)})" if x.get("also_with") else ""
|
|
263
|
+
what = f"Files in `{x['path']}` changed" if x.get("folder") else f"`{x['path']}` changed"
|
|
264
|
+
return (f"{what} in {x['together']} of the {x['changes']} commits that changed `{x['file']}`{also}"
|
|
265
|
+
+ (f"; {tail}" if tail else ""))
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _list(xs: list[str], k: int) -> str:
|
|
269
|
+
return ", ".join(f"`{x}`" for x in xs[:k]) + (f" and {len(xs) - k} more" if len(xs) > k else "")
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def about(run: dict) -> str:
|
|
273
|
+
"""Where the numbers come from, said once above the lines."""
|
|
274
|
+
bulk = f", leaving out {run['bulk']} that changed more than {BULK} files" if run.get("bulk") else ""
|
|
275
|
+
return f"from the last {run['commits']} commits{bulk}"
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
# -- the spec loop and pull requests -------------------------------------------------------------------
|
|
279
|
+
PATHISH = re.compile(r"[\w.@+-]+(?:/[\w.@+-]+)*\.[A-Za-z0-9]+|[\w.@+-]+(?:/[\w.@+-]+)+/?")
|
|
280
|
+
# PATHISH tries every place a dot could end a run, from every place it could start: quadratic in a run's length, so a
|
|
281
|
+
# spec or pull request description with one long word would hold the plan for minutes. It only ever matches inside a
|
|
282
|
+
# run of these characters, so it is tried run by run, and a run longer than any path is not one.
|
|
283
|
+
_RUN = re.compile(r"[\w.@+/-]+")
|
|
284
|
+
LONGEST_PATH = 400
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def written_paths(texts: Iterable[str], known: Iterable[str]) -> set:
|
|
288
|
+
"""The known paths that some text names: the whole path or its end (`validation/local.ts`, `EDITOR_GUIDE.md`),
|
|
289
|
+
in backticks or not, or a folder they are in (`cases/`). A name that ends several paths names them all."""
|
|
290
|
+
words = _words(texts)
|
|
291
|
+
if not words:
|
|
292
|
+
return set()
|
|
293
|
+
folders = {w for w in words if "/" in w}
|
|
294
|
+
out = set()
|
|
295
|
+
for p in known:
|
|
296
|
+
parts = p.split("/")
|
|
297
|
+
if p in words or any("/".join(parts[k:]) in words for k in range(1, len(parts))):
|
|
298
|
+
out.add(p)
|
|
299
|
+
elif any(("/" + "/".join(parts[:k])).endswith("/" + w) for w in folders for k in range(1, len(parts))):
|
|
300
|
+
out.add(p) # a folder the text names covers what is in it
|
|
301
|
+
return out
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _words(texts: Iterable[str]) -> set:
|
|
305
|
+
words = {w.strip("`'\"()[],;:").lstrip("./").rstrip("/") for t in texts for run in _RUN.findall(t or "")
|
|
306
|
+
if len(run) <= LONGEST_PATH for w in PATHISH.findall(run)}
|
|
307
|
+
return {w for w in words if w}
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def known_paths(con, run: dict) -> list[str]:
|
|
311
|
+
return [r[0] for r in con.execute("SELECT path FROM coupling_files WHERE repo_id = ? AND sha = ?", (run["repo"], run["sha"]))]
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _covered(named: set):
|
|
315
|
+
"""A path is covered when it is named, and a folder when something named is in it (a new file or folder too:
|
|
316
|
+
`tests/cases/missing-ok/` covers `tests/cases/`)."""
|
|
317
|
+
return lambda p: p in named or (p.endswith("/") and any(n.startswith(p) for n in named))
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def for_spec(con, names, links: list[dict], limit: int = 30) -> dict:
|
|
321
|
+
"""For the files a spec's tasks touch (the files of the code they name, and files they name by path), the files
|
|
322
|
+
that usually change with them that no task names. {"about", "files", "total"}, or {} with no history to read."""
|
|
323
|
+
texts = [l.get("text") or "" for l in links]
|
|
324
|
+
by_repo: dict[str, set] = {}
|
|
325
|
+
for l in links:
|
|
326
|
+
ids = list(l.get("nodes") or []) + list(l.get("into") or []) + [n["parent"] for n in l.get("new") or [] if n.get("parent")]
|
|
327
|
+
for i in ids:
|
|
328
|
+
r = names.by_id.get(i)
|
|
329
|
+
if r is not None and r["path"]:
|
|
330
|
+
by_repo.setdefault(i.split(":", 1)[0], set()).add(r["path"])
|
|
331
|
+
out, runs, fns = [], [], []
|
|
332
|
+
for repo, root in store.roots(con).items():
|
|
333
|
+
if not Path(root).is_dir():
|
|
334
|
+
continue
|
|
335
|
+
try:
|
|
336
|
+
run = ensure(con, repo, root)
|
|
337
|
+
except Exception: # history is a lead, never a reason for the plan to fail
|
|
338
|
+
run = None
|
|
339
|
+
if run is None:
|
|
340
|
+
continue
|
|
341
|
+
named = written_paths(texts, known_paths(con, run))
|
|
342
|
+
touched = sorted(by_repo.get(repo, set()) | named)
|
|
343
|
+
if not touched:
|
|
344
|
+
continue
|
|
345
|
+
runs.append(run)
|
|
346
|
+
written = {w + "/" for w in _words(texts) if "/" in w} # paths not in the history yet, such as a new folder
|
|
347
|
+
out += [{**x, "repo": repo} for x in missed(con, run, touched, _covered(named | set(touched) | written))]
|
|
348
|
+
# Functions that usually changed with the functions the tasks name, and that no task names.
|
|
349
|
+
tasked = [i for l in links for i in l.get("nodes") or [] if i.split(":", 1)[0] == repo]
|
|
350
|
+
fns += _functions(con, repo, root, run["sha"], tasked, tasked)
|
|
351
|
+
if not runs:
|
|
352
|
+
return {}
|
|
353
|
+
out.sort(key=lambda x: (-x["confidence"], -x["together"], bool(x.get("folder")), x["path"]))
|
|
354
|
+
return {"about": about(_merged(runs)), "files": out[:limit], "total": len(out), "functions": fns[:limit]}
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
MAX_FUNCTIONS = 6 # functions whose own history is read, per plan or pull request (each costs a few parses)
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _functions(con, repo: str, root, sha: str, targets: list[str], named: Iterable[str]) -> list[dict]:
|
|
361
|
+
"""leyline.fncoupling for up to MAX_FUNCTIONS of `targets`, leaving out partners `named` covers (the function
|
|
362
|
+
itself, or something it sits in or that sits in it)."""
|
|
363
|
+
from . import fncoupling
|
|
364
|
+
named = set(named)
|
|
365
|
+
|
|
366
|
+
def covered(g):
|
|
367
|
+
return any(g == n or g.startswith(n + ".") or g.startswith(n + "/") or n.startswith(g + ".") for n in named)
|
|
368
|
+
targets = [i for i in dict.fromkeys(targets) if (con.execute("SELECT kind FROM nodes WHERE id = ?", (i,)).fetchone()
|
|
369
|
+
or [None])[0] in fncoupling.KINDS][:MAX_FUNCTIONS]
|
|
370
|
+
if not targets:
|
|
371
|
+
return []
|
|
372
|
+
r = fncoupling.missed(con, repo, Path(root), sha, targets, covered)
|
|
373
|
+
return [{**x, "repo": repo} for x in r.get("functions") or []]
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def changed_since(root: Path, base: str) -> list[str]:
|
|
377
|
+
"""Files the checkout changed since `base`: committed, uncommitted and new."""
|
|
378
|
+
out = set()
|
|
379
|
+
for args in (("diff", "--relative", "--name-only", "-z", "--no-renames", base),
|
|
380
|
+
("ls-files", "-z", "--others", "--exclude-standard")):
|
|
381
|
+
listed = _git(root, "-c", "core.quotePath=false", *args)
|
|
382
|
+
out |= {p.decode("utf-8", "replace") for p in (listed or b"").split(b"\0") if p}
|
|
383
|
+
return sorted(p for p in out if not p.startswith(".leyline/"))
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def for_pr(con, repo: str, root: Path, base_sha: str, limit: int = 30, functions: Iterable[str] = (),
|
|
387
|
+
changed_ids: Iterable[str] = ()) -> dict:
|
|
388
|
+
"""For the files a branch changed, the files that usually changed with them in the history before the branch and
|
|
389
|
+
that the branch did not change. {"about", "files", "total", "functions"}, or {} with no history to read. With
|
|
390
|
+
`functions` (functions the branch edited), also the functions that usually changed with them and that are not in
|
|
391
|
+
`changed_ids`."""
|
|
392
|
+
try:
|
|
393
|
+
run = ensure(con, repo, root, base_sha)
|
|
394
|
+
except Exception:
|
|
395
|
+
run = None
|
|
396
|
+
if run is None:
|
|
397
|
+
return {}
|
|
398
|
+
changed = changed_since(Path(root), base_sha)
|
|
399
|
+
found = missed(con, run, changed, _covered(set(changed)))
|
|
400
|
+
fns = _functions(con, repo, root, base_sha, list(functions), set(functions) | set(changed_ids)) if functions else []
|
|
401
|
+
return {"about": about(run) + " before the branch", "files": found[:limit], "total": len(found), "functions": fns[:limit]}
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _merged(runs: list[dict]) -> dict:
|
|
405
|
+
return {"commits": sum(r["commits"] for r in runs), "bulk": sum(r["bulk"] for r in runs)}
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
# -- the coupling command and tool ------------------------------------------------------------------------------
|
|
409
|
+
def query(con, path: Optional[str] = None, min_together: int = MIN_TOGETHER, min_confidence: float = MIN_CONFIDENCE,
|
|
410
|
+
limit: int = 20, cwd: Optional[Path] = None) -> dict:
|
|
411
|
+
"""A file's partners (files, and folders no single file explains), or with no path the most coupled pairs."""
|
|
412
|
+
held = store.roots(con)
|
|
413
|
+
roots = {r: Path(p) for r, p in held.items() if Path(p).is_dir()}
|
|
414
|
+
if not roots:
|
|
415
|
+
if held:
|
|
416
|
+
return {"error": "no repository in the store is where it was mapped (" + ", ".join(
|
|
417
|
+
f"{r} at {p}" for r, p in sorted(held.items())) + "); map it again from where it is now"}
|
|
418
|
+
return {"error": "the store does not say where any repository is; run `leyline map` first"}
|
|
419
|
+
runs = {}
|
|
420
|
+
for repo, root in roots.items():
|
|
421
|
+
run = ensure(con, repo, root)
|
|
422
|
+
if run is not None:
|
|
423
|
+
runs[repo] = run
|
|
424
|
+
if not runs:
|
|
425
|
+
if any(_git(root, "rev-parse", "--git-dir") is not None for root in roots.values()):
|
|
426
|
+
return {"error": "the repository has no commits yet, so there is no history to read"}
|
|
427
|
+
return {"error": "no repository in the store is under git, so there is no history to read"}
|
|
428
|
+
thresholds = {"min_together": min_together, "min_confidence": min_confidence}
|
|
429
|
+
if not path:
|
|
430
|
+
pairs = [{**x, "repo": repo} for repo, run in runs.items() for x in strongest(con, run, limit=limit, **thresholds)]
|
|
431
|
+
pairs.sort(key=lambda x: (-x["together"], -x["confidence"]))
|
|
432
|
+
return {"about": about(_merged(list(runs.values()))), "pairs": pairs[:limit], **thresholds}
|
|
433
|
+
hits = _find(con, runs, roots, path, cwd or Path.cwd())
|
|
434
|
+
if not hits:
|
|
435
|
+
return {"error": f"no file {path!r} in the history read ({about(_merged(list(runs.values())))}); give its path"
|
|
436
|
+
" in the repository"}
|
|
437
|
+
if len(hits) > 1:
|
|
438
|
+
return {"error": f"{path!r} could be {len(hits)} files: " + ", ".join(p for _, p in hits[:8]) + "; give more of the path"}
|
|
439
|
+
repo, p = hits[0]
|
|
440
|
+
run = runs[repo]
|
|
441
|
+
row = con.execute("SELECT changes FROM coupling_files WHERE repo_id = ? AND sha = ? AND path = ?", (repo, run["sha"], p)).fetchone()
|
|
442
|
+
ps = partners(con, run, [p], folders=True, **thresholds)[p]
|
|
443
|
+
return {"repo": repo, "path": p, "changes": row[0] if row else 0, "about": about(run), **thresholds,
|
|
444
|
+
"partners": ps[:limit], "total": len(ps)}
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def _find(con, runs: dict, roots: dict, path: str, cwd: Path) -> list[tuple[str, str]]:
|
|
448
|
+
"""(repo, path) pairs a written path can be: relative to here, to a repository, or the end of a path."""
|
|
449
|
+
given = Path(path)
|
|
450
|
+
full = given if given.is_absolute() else cwd / given
|
|
451
|
+
for repo, root in roots.items():
|
|
452
|
+
try:
|
|
453
|
+
rel = full.resolve().relative_to(root.resolve()).as_posix()
|
|
454
|
+
except (ValueError, OSError):
|
|
455
|
+
continue
|
|
456
|
+
if repo in runs and rel in set(known_paths(con, runs[repo])):
|
|
457
|
+
return [(repo, rel)]
|
|
458
|
+
name = path.strip().lstrip("./")
|
|
459
|
+
exact = [(repo, name) for repo, run in runs.items() if name in set(known_paths(con, run))]
|
|
460
|
+
if exact:
|
|
461
|
+
return exact
|
|
462
|
+
return [(repo, p) for repo, run in runs.items() for p in known_paths(con, run) if p.endswith("/" + name)]
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def text(r: dict) -> str:
|
|
466
|
+
"""The coupling command's answer, for a person."""
|
|
467
|
+
if "error" in r:
|
|
468
|
+
return "leyline: " + r["error"]
|
|
469
|
+
rule = f"at least {r['min_together']} commits together, and {round(r['min_confidence'] * 100)}% of the first file's"
|
|
470
|
+
if "pairs" in r:
|
|
471
|
+
if not r["pairs"]:
|
|
472
|
+
return f"No two files usually change together ({r['about']}; {rule})."
|
|
473
|
+
L = [f"Files that usually change together ({r['about']}; {rule}):", ""]
|
|
474
|
+
L += [f"- {line(x)}" for x in r["pairs"]]
|
|
475
|
+
return "\n".join(L)
|
|
476
|
+
if not r["partners"]:
|
|
477
|
+
own = rule.replace("the first file's", "its")
|
|
478
|
+
return (f"`{r['path']}` changed in {r['changes']} commits ({r['about']}); nothing usually changed with it"
|
|
479
|
+
f" ({own}).")
|
|
480
|
+
L = [f"`{r['path']}` changed in {r['changes']} commits ({r['about']}). What usually changed with it:", ""]
|
|
481
|
+
L += [f"- {'files in ' if x.get('folder') else ''}`{x['path']}`: {x['together']} of {x['changes']}"
|
|
482
|
+
f" ({round(x['confidence'] * 100)}%)" for x in r["partners"]]
|
|
483
|
+
if r["total"] > len(r["partners"]):
|
|
484
|
+
L.append(f"- and {r['total'] - len(r['partners'])} more (--limit)")
|
|
485
|
+
return "\n".join(L)
|