leyline-code 0.8.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. leyline/__init__.py +3 -0
  2. leyline/adapters/__init__.py +19 -0
  3. leyline/adapters/csharp.py +1026 -0
  4. leyline/adapters/generic.py +844 -0
  5. leyline/adapters/python.py +929 -0
  6. leyline/adapters/typescript.py +1512 -0
  7. leyline/affected.py +509 -0
  8. leyline/agent_skills.py +311 -0
  9. leyline/change.py +500 -0
  10. leyline/channels.py +1782 -0
  11. leyline/cli.py +1029 -0
  12. leyline/cluster.py +149 -0
  13. leyline/context.py +531 -0
  14. leyline/coupling.py +485 -0
  15. leyline/coverage.py +376 -0
  16. leyline/diagrams.py +680 -0
  17. leyline/diff.py +1078 -0
  18. leyline/drift.py +747 -0
  19. leyline/exact.py +410 -0
  20. leyline/explain.py +973 -0
  21. leyline/export.py +673 -0
  22. leyline/fncoupling.py +328 -0
  23. leyline/formats.py +923 -0
  24. leyline/grade.py +207 -0
  25. leyline/incremental.py +931 -0
  26. leyline/indexer.py +3733 -0
  27. leyline/learnings.py +743 -0
  28. leyline/loop.py +584 -0
  29. leyline/model.py +142 -0
  30. leyline/outline.py +1066 -0
  31. leyline/packing.py +86 -0
  32. leyline/patterns.py +478 -0
  33. leyline/pr.py +1249 -0
  34. leyline/props.py +255 -0
  35. leyline/query.py +683 -0
  36. leyline/quick.py +853 -0
  37. leyline/related.py +266 -0
  38. leyline/removal.py +231 -0
  39. leyline/renames.py +231 -0
  40. leyline/rereview.py +375 -0
  41. leyline/roslyn_refs/Program.cs +179 -0
  42. leyline/roslyn_refs/RoslynRefs.csproj +16 -0
  43. leyline/rules.py +171 -0
  44. leyline/schema.sql +178 -0
  45. leyline/scip_pb2.py +97 -0
  46. leyline/server.py +1230 -0
  47. leyline/skills/leyline-adversarial-review/SKILL.md +121 -0
  48. leyline/skills/leyline-ask/SKILL.md +57 -0
  49. leyline/skills/leyline-change-impact/SKILL.md +79 -0
  50. leyline/skills/leyline-explain-flow/SKILL.md +70 -0
  51. leyline/skills/leyline-explore-module/SKILL.md +55 -0
  52. leyline/skills/leyline-pr-review/SKILL.md +77 -0
  53. leyline/skills/leyline-quick-change/SKILL.md +63 -0
  54. leyline/skills/leyline-spec/SKILL.md +179 -0
  55. leyline/skills/leyline-tour/SKILL.md +46 -0
  56. leyline/spec.py +1819 -0
  57. leyline/store.py +560 -0
  58. leyline/tours.py +352 -0
  59. leyline/verdicts.py +390 -0
  60. leyline/viewer/viewer.html +1622 -0
  61. leyline_code-0.8.1.dist-info/METADATA +1325 -0
  62. leyline_code-0.8.1.dist-info/RECORD +65 -0
  63. leyline_code-0.8.1.dist-info/WHEEL +4 -0
  64. leyline_code-0.8.1.dist-info/entry_points.txt +2 -0
  65. leyline_code-0.8.1.dist-info/licenses/LICENSE +674 -0
leyline/outline.py ADDED
@@ -0,0 +1,1066 @@
1
+ """Outline: a large module split into parts a person can hold in their head, and names for those parts.
2
+
3
+ On a large repository a module is still huge (Parlance's `editor/client` holds 249 files) and the map draws it as one
4
+ box. `outline(con, part)` splits one level of it into at most MAX_PARTS parts:
5
+
6
+ folders when the folder tree means something: each subfolder is a part, and the files lying loose beside
7
+ them are one more ("files in src/"). A folder that holds nearly everything is shown instead of its
8
+ parent, so `editor/client` is shown from `editor/client/src`.
9
+ groups where a folder is flat (MIN_CLUSTER_FILES loose files or more), its files are grouped by Louvain
10
+ community detection over calls, type use and inheritance between their types and functions; each
11
+ file goes to the group that holds most of its code, so a group is a set of whole files. When few
12
+ of them link to each other (a folder of tests), they are grouped by the code they use instead. A
13
+ module that is one flat folder reuses the systems the map proposed (cluster.py).
14
+ files a small folder, a group or what is left over: one part per file.
15
+ a leaf a file, a type or a proposed system: its key types and functions.
16
+
17
+ A level of more than MAX_PARTS parts shows the largest and one `@more` part holding the rest. For each part it gives
18
+ its size, its entry points, the parts and modules it uses and that use it (counted links), the channels that cross
19
+ its edge, its busiest functions (most flows through them) and what makes it risky beside its siblings.
20
+
21
+ Part ids hold across maps where they can: a module is its node id, a folder `<repo>:dir:<path>`, the loose files of a
22
+ folder `<repo>:dir:<path>#files`, a group `<repo>:dir:<path>#system:<anchor>` named after its most linked type or
23
+ file, what no group took `<repo>:dir:<path>#other`, a file or a type its node id.
24
+
25
+ `name_part` stores a name and a one-line summary for a part (table `part_names`, inferred layer unless the person
26
+ said it), with the part's files at the time. Outlines, the map page, `overview` and `explain_path` show the names. A
27
+ name whose part kept less than half of those files, or most of whose evidence is gone, is shown as "may be stale"; a
28
+ group whose anchor changed (so its id did) takes the name of the old group of the same folder it shares most files
29
+ with.
30
+ """
31
+
32
+ from __future__ import annotations
33
+
34
+ import json
35
+ import posixpath
36
+ import re
37
+ import threading
38
+ import time
39
+ from collections import Counter, defaultdict
40
+ from dataclasses import dataclass, field
41
+ from typing import Optional
42
+
43
+ from . import cluster, store
44
+
45
+ MAX_PARTS = 12
46
+ MIN_CLUSTER_FILES = 8 # a folder with fewer loose files lists them as they are
47
+ MIN_CLUSTER_FILES_BESIDE = 15 # the same, for loose files beside subfolders: fewer are one part, "files in x/"
48
+ MIN_MODULARITY = 0.3 # as cluster.py: a weaker split is noise
49
+ NEARLY_ALL = 0.9 # a folder holding this share of a part's files, with at most SMALL_REST others, is shown instead
50
+ SMALL_REST = 5
51
+ KEPT_FOR_FRESH = 0.5 # a named part that kept less than this share of its files may be stale
52
+ TOP = ("<module>", "<top-level>")
53
+ TYPE_LINKS = ("uses_type", "instantiates", "extends", "implements")
54
+
55
+ ENTRY_WORDS = { # how each kind of entry point is said, one and many
56
+ "route": ("route", "routes"), "message": ("message handler", "message handlers"),
57
+ "entry": ("program", "programs"), "command": ("command", "commands"), "ui": ("UI handler", "UI handlers"),
58
+ "engine": ("engine callback", "engine callbacks"), "component": ("UI component", "UI components"),
59
+ }
60
+ ENGINE_CALLBACKS = {"Awake", "Start", "Update", "FixedUpdate", "LateUpdate", "OnEnable", "_ready", "_process",
61
+ "_physics_process", "_input"}
62
+
63
+
64
+ # -- the model: everything an outline reads, loaded once per map run ----------------------------------------------------
65
+ class Model:
66
+ def __init__(self, con):
67
+ q = con.execute
68
+ self.repos = [r[0] for r in q("SELECT id FROM nodes WHERE kind = 'repo' ORDER BY id")]
69
+ self.modules: dict[str, dict] = {}
70
+ for r in q("SELECT id, name, path, repo_id FROM nodes WHERE kind = 'module' AND layer = 'fact'"):
71
+ self.modules[r[0]] = {"name": r[1], "path": "" if r[2] in (None, ".") else r[2], "repo": r[3]}
72
+ self.files: dict[str, dict] = {}
73
+ self.files_of_module: dict[str, list] = defaultdict(list)
74
+ for r in q("SELECT id, path, parent_id, repo_id, span_end FROM nodes WHERE kind = 'file' AND layer = 'fact'"
75
+ " ORDER BY path"):
76
+ mod = r[2] if r[2] in self.modules else None
77
+ self.files[r[0]] = {"path": r[1] or "", "module": mod, "repo": r[3], "lines": r[4] or 0}
78
+ if mod:
79
+ self.files_of_module[mod].append(r[0])
80
+ self.file_by_path = {(f["repo"], f["path"]): i for i, f in self.files.items()}
81
+ nodes = {r[0]: r for r in q("SELECT id, kind, name, parent_id, attrs, span_start, span_end FROM nodes"
82
+ " WHERE kind IN ('type', 'callable', 'test') AND layer = 'fact'")}
83
+ self.name = {i: r[2] for i, r in nodes.items()}
84
+ self.kind = {i: r[1] for i, r in nodes.items()}
85
+ self.lines_of = {i: (r[6] or 0) - (r[5] or 0) + 1 for i, r in nodes.items()}
86
+ self.file_of: dict[str, str] = {}
87
+ for nid, fid in q("SELECT node_id, file_id FROM ancestry WHERE file_id IS NOT NULL"):
88
+ if nid in nodes and fid in self.files:
89
+ self.file_of[nid] = fid
90
+ # The unit of a node: its outermost type, or its file when it sits loose in one (as cluster.py has it).
91
+ self.unit: dict[str, str] = {}
92
+ for i in nodes:
93
+ cur = i
94
+ while nodes.get(nodes[cur][3]) is not None:
95
+ cur = nodes[cur][3]
96
+ self.unit[i] = cur if nodes[cur][1] == "type" else self.file_of.get(i)
97
+ self.units_of_file: dict[str, list] = defaultdict(list)
98
+ self.code_units: set = set()
99
+ for i, u in self.unit.items():
100
+ if u and nodes[i][1] in ("type", "callable"):
101
+ self.code_units.add(u)
102
+ for u in sorted(self.code_units):
103
+ f = self.file_of.get(u) or (u if u in self.files else None)
104
+ if f:
105
+ self.units_of_file[f].append(u)
106
+ entry = {r[0]: json.loads(r[1] or "{}").get("trigger") for r in q(
107
+ "SELECT e.dst_id, n.attrs FROM edges e JOIN nodes n ON n.id = e.src_id WHERE e.kind = 'exposes'")}
108
+ channel_in: dict[str, set] = defaultdict(set)
109
+ self.channels: dict[str, list] = defaultdict(list) # file -> (src file, dst file, channel, address) touching it
110
+ for s, d, a in q("SELECT src_id, dst_id, attrs FROM edges WHERE kind = 'communicates'"):
111
+ try:
112
+ at = json.loads(a or "{}")
113
+ except ValueError:
114
+ at = {}
115
+ ch = at.get("channel") or "channel"
116
+ channel_in[d].add(ch)
117
+ fs, fd = self._file(s), self._file(d)
118
+ if fs and fd and fs != fd:
119
+ addr = str(at.get("address") or "")
120
+ if re.match(r"^[\w.-]+:[\w#-]+:", addr): # an event's address is its node id: say its name
121
+ addr = re.split(r"[.:]", addr)[-1]
122
+ c = (fs, fd, ch, addr)
123
+ self.channels[fs].append(c)
124
+ self.channels[fd].append(c)
125
+ # Functions and tests per file, and what kind of entry point each function is.
126
+ self.functions: dict[str, list] = defaultdict(list)
127
+ self.tests: dict[str, int] = Counter()
128
+ self.entry_kind: dict[str, str] = {}
129
+ self.route: dict[str, str] = {}
130
+ for i, r in nodes.items():
131
+ f = self.file_of.get(i)
132
+ if f is None or r[1] == "type":
133
+ continue
134
+ a = _json(r[4])
135
+ if r[1] == "test" or a.get("is_test"):
136
+ self.tests[f] += 1
137
+ continue
138
+ if r[2] in TOP and i not in entry:
139
+ continue
140
+ self.functions[f].append(i)
141
+ parent = nodes.get(r[3])
142
+ k = _entry_kind(i, r[2], a, parent, entry, channel_in.get(i, set()))
143
+ if k:
144
+ self.entry_kind[i] = k
145
+ if a.get("route"):
146
+ self.route[i] = a["route"]
147
+ self.flows = {r[0]: r[1] for r in q("SELECT k.id, COUNT(*) FROM steps s JOIN keys k ON k.k = s.callable"
148
+ " GROUP BY s.callable")}
149
+ self.tested = {r[0] for r in q(store.TESTED)}
150
+ # Links between files (for uses / used by) and between units of one module (for grouping).
151
+ self.fout: dict[str, Counter] = defaultdict(Counter)
152
+ self.fin: dict[str, Counter] = defaultdict(Counter)
153
+ self.uadj: dict[str, Counter] = defaultdict(Counter)
154
+
155
+ def link(s, d, n, w):
156
+ fs, fd = self._file(s), self._file(d)
157
+ if fs and fd and fs != fd:
158
+ self.fout[fs][fd] += n
159
+ self.fin[fd][fs] += n
160
+ a, b = self.unit.get(s) or (s if s in self.files else None), self.unit.get(d) or (d if d in self.files else None)
161
+ if a and b and a != b and fs and fd and self.files[fs]["module"] == self.files[fd]["module"]:
162
+ self.uadj[a][b] += w
163
+ self.uadj[b][a] += w
164
+ for s, d, n in q("SELECT sk.id, dk.id, c.n FROM (SELECT src, dst, COUNT(*) AS n FROM call_sites GROUP BY src, dst) c"
165
+ " JOIN keys sk ON sk.k = c.src JOIN keys dk ON dk.k = c.dst"):
166
+ link(s, d, n, cluster.WEIGHTS["calls"] * n)
167
+ for k, s, d in q(f"SELECT kind, src_id, dst_id FROM edges WHERE kind IN ({','.join('?' * len(TYPE_LINKS))})",
168
+ TYPE_LINKS):
169
+ link(s, d, 1, cluster.WEIGHTS[k])
170
+ # Systems the map proposed (cluster.py), by module.
171
+ self.systems: dict[str, list] = defaultdict(list)
172
+ self.system_units: dict[str, list] = defaultdict(list)
173
+ self.system_name: dict[str, str] = {}
174
+ for sid, name, mod in q("SELECT id, name, parent_id FROM nodes WHERE kind = 'system' ORDER BY id"):
175
+ self.systems[mod].append(sid)
176
+ self.system_name[sid] = name
177
+ for s, d in q("SELECT src_id, dst_id FROM edges WHERE kind = 'groups'"):
178
+ self.system_units[s].append(d)
179
+ self.splits: dict[str, tuple] = {}
180
+ self.lock = threading.Lock()
181
+
182
+ def _file(self, nid: str) -> Optional[str]:
183
+ return nid if nid in self.files else self.file_of.get(nid)
184
+
185
+ def lines(self, files) -> int:
186
+ return sum(self.files[f]["lines"] for f in files)
187
+
188
+
189
+ def _json(raw) -> dict:
190
+ try:
191
+ return json.loads(raw or "{}")
192
+ except ValueError:
193
+ return {}
194
+
195
+
196
+ def _entry_kind(nid, name, a, parent, entry, chans) -> Optional[str]:
197
+ """What kind of entry point a function is, as explain.find_flows says it; None for an ordinary function."""
198
+ if a.get("native_kind") == "route_handler" or a.get("route") or "http" in chans:
199
+ return "route"
200
+ if chans & {"queue", "event", "rpc"}:
201
+ return "message"
202
+ if nid in entry:
203
+ return "entry"
204
+ decorators = " ".join(map(str, a.get("decorators") or []))
205
+ if re.search(r"\.command\b|\bcommand\(|\bcli\b", decorators):
206
+ return "command"
207
+ if re.match(r"(on|handle)[A-Z_]", name or ""):
208
+ return "ui"
209
+ if name in ENGINE_CALLBACKS:
210
+ return "engine"
211
+ if a.get("native_kind") == "component":
212
+ return "component"
213
+ if parent is not None and parent[1] == "callable" and "component" in (parent[4] or ""):
214
+ return "ui"
215
+ return None
216
+
217
+
218
+ _cache: dict = {}
219
+ _cache_lock = threading.Lock()
220
+
221
+
222
+ def model(con) -> Model:
223
+ """The model of a store, built once per map run and kept."""
224
+ gen = con.execute("SELECT value FROM meta WHERE key = 'generation'").fetchone()
225
+ stamp = (gen[0] if gen else None, con.execute("SELECT COUNT(*) FROM nodes").fetchone()[0],
226
+ con.execute("SELECT COUNT(*) FROM call_sites").fetchone()[0])
227
+ key = str(store.store_file(con))
228
+ with _cache_lock:
229
+ hit = _cache.get(key)
230
+ if hit is not None and hit[0] == stamp and stamp[0] is not None:
231
+ return hit[1]
232
+ m = Model(con)
233
+ with _cache_lock:
234
+ _cache[key] = (stamp, m)
235
+ return m
236
+
237
+
238
+ # -- parts ---------------------------------------------------------------------------------------------------------------
239
+ @dataclass
240
+ class Part:
241
+ id: str
242
+ kind: str # workspace, repo, module, folder, files, system, other, file, type, more
243
+ label: str # its name until someone gives it one
244
+ files: list # file ids, sorted
245
+ path: str = "" # the folder it stands for (a file's path for a file)
246
+ module: str = ""
247
+ repo: str = ""
248
+ parent: str = "" # the id of the part it was split from
249
+ rest: list = field(default_factory=list) # a `more` part: the parts it holds
250
+ around: list = field(default_factory=list) # a group made by what it uses: the names of that code
251
+ node: str = "" # a type: its node id
252
+
253
+
254
+ def _dir_id(repo: str, path: str) -> str:
255
+ return f"{repo}:dir:{path}"
256
+
257
+
258
+ def _rel(m: Model, part: Part, path: str) -> str:
259
+ """A folder's path as shown inside its module: relative to the module's own folder."""
260
+ base = m.modules.get(part.module, {}).get("path", "")
261
+ return path[len(base) + 1:] if base and path.startswith(base + "/") else path
262
+
263
+
264
+ def _under(top: str, path: str) -> str:
265
+ """A folder's path as shown inside the part (of folder `top`) it was split from."""
266
+ if path == top:
267
+ return (posixpath.basename(path) or "(repository root)") + "/"
268
+ return (path[len(top) + 1:] if top and path.startswith(top + "/") else path) + "/"
269
+
270
+
271
+ def _folder_label(m: Model, mod: str, path: str) -> str:
272
+ base = m.modules.get(mod, {}).get("path", "")
273
+ if path == base:
274
+ return (base or "(repository root)") + "/"
275
+ return (path[len(base) + 1:] if base and path.startswith(base + "/") else path) + "/"
276
+
277
+
278
+ def module_part(m: Model, mod: str) -> Part:
279
+ info = m.modules[mod]
280
+ return Part(mod, "module", (info["path"] or "(repository root)") + "/", list(m.files_of_module.get(mod, [])),
281
+ info["path"], mod, info["repo"])
282
+
283
+
284
+ def folder_part(m: Model, repo: str, path: str, loose_only: bool = False) -> Optional[Part]:
285
+ """The files under a folder (or lying directly in it), in the module that holds the folder."""
286
+ path = path.strip("/")
287
+ mods = [k for k, v in m.modules.items() if v["repo"] == repo and (not v["path"] or path == v["path"]
288
+ or path.startswith(v["path"] + "/"))]
289
+ if not mods:
290
+ return None
291
+ mod = max(mods, key=lambda k: len(m.modules[k]["path"]))
292
+ pre = path + "/" if path else ""
293
+ files = [f for f in m.files_of_module.get(mod, []) if m.files[f]["path"].startswith(pre)
294
+ and (not loose_only or "/" not in m.files[f]["path"][len(pre):])]
295
+ if not files:
296
+ return None
297
+ if loose_only:
298
+ name = "files in " + _folder_label(m, mod, path)
299
+ return Part(_dir_id(repo, path) + "#files", "files", name, files, path, mod, repo)
300
+ return Part(_dir_id(repo, path), "folder", _folder_label(m, mod, path), files, path, mod, repo)
301
+
302
+
303
+ def file_part(m: Model, f: str, parent: str = "") -> Part:
304
+ info = m.files[f]
305
+ return Part(f, "file", info["path"].rsplit("/", 1)[-1], [f], info["path"], info["module"] or "", info["repo"], parent)
306
+
307
+
308
+ def _sort(m: Model, parts: list) -> list:
309
+ return sorted(parts, key=lambda p: (-m.lines(p.files), -len(p.files), p.id))
310
+
311
+
312
+ def _pack(m: Model, owner: Part, kids: list) -> list:
313
+ """At most MAX_PARTS, largest first; the rest in one `more` part."""
314
+ kids = _sort(m, kids)
315
+ if len(kids) <= MAX_PARTS:
316
+ return kids
317
+ keep, rest = kids[:MAX_PARTS - 1], kids[MAX_PARTS - 1:]
318
+ files = sorted({f for k in rest for f in k.files})
319
+ more = Part(owner.id + "@more", "more", f"{len(rest)} more parts", files, owner.path, owner.module, owner.repo,
320
+ owner.id, rest=rest)
321
+ return _sort(m, keep) + [more]
322
+
323
+
324
+ def split(m: Model, part: Part) -> tuple[list, str, Optional[str]]:
325
+ """The parts one level down: (parts, how they were split, the folder shown from when it is not the part's own)."""
326
+ with m.lock:
327
+ hit = m.splits.get(part.id)
328
+ if hit is not None:
329
+ return hit
330
+ out = _split(m, part)
331
+ with m.lock:
332
+ m.splits[part.id] = out
333
+ return out
334
+
335
+
336
+ def _split(m: Model, part: Part) -> tuple[list, str, Optional[str]]:
337
+ if part.kind == "workspace":
338
+ return [Part(r, "repo", r, sorted(f for f, v in m.files.items() if v["repo"] == r), "", "", r, part.id)
339
+ for r in m.repos], "repositories", None
340
+ if part.kind == "repo":
341
+ mods = [module_part(m, k) for k, v in m.modules.items() if v["repo"] == part.repo and m.files_of_module.get(k)]
342
+ for x in mods:
343
+ x.parent = part.id
344
+ return _pack(m, part, mods), "modules", None
345
+ if part.kind == "more":
346
+ return part.rest, "the rest of the level", None
347
+ if part.kind in ("file", "type"):
348
+ return [], "", None
349
+ if part.kind in ("system", "files", "other"):
350
+ if len(part.files) < 2:
351
+ return [], "", None
352
+ return _pack(m, part, [file_part(m, f, part.id) for f in part.files]), "files", None
353
+ return _folders(m, part)
354
+
355
+
356
+ def _folders(m: Model, part: Part) -> tuple[list, str, Optional[str]]:
357
+ base, files, extra, shown_from = part.path, list(part.files), [], None
358
+ while True:
359
+ pre = base + "/" if base else ""
360
+ subs: dict[str, list] = defaultdict(list)
361
+ loose = []
362
+ for f in files:
363
+ rest = m.files[f]["path"][len(pre):]
364
+ (subs[rest.split("/", 1)[0]].append(f) if "/" in rest else loose.append(f))
365
+ if not subs:
366
+ break
367
+ big = max(subs, key=lambda k: (len(subs[k]), k))
368
+ others = len(files) - len(subs[big])
369
+ if len(subs[big]) < NEARLY_ALL * len(files) or others > SMALL_REST:
370
+ break
371
+ # One folder holds nearly everything: show it, and keep the few files beside it as parts of their own.
372
+ for k, fs in sorted(subs.items()):
373
+ if k != big:
374
+ extra.append(Part(_dir_id(part.repo, pre + k), "folder", _under(part.path, pre + k),
375
+ fs, pre + k, part.module, part.repo, part.id))
376
+ if len(loose) == 1:
377
+ extra.append(file_part(m, loose[0], part.id))
378
+ elif loose:
379
+ extra.append(Part(_dir_id(part.repo, base) + "#files", "files", "files in " + _under(part.path, base),
380
+ loose, base, part.module, part.repo, part.id))
381
+ base, files = pre + big, subs[big]
382
+ shown_from = base
383
+ kids = list(extra)
384
+ for k, fs in sorted(subs.items()):
385
+ kids.append(Part(_dir_id(part.repo, pre + k), "folder", _under(part.path, pre + k), fs, pre + k,
386
+ part.module, part.repo, part.id))
387
+ how = "folders" if subs else "files"
388
+ # Beside folders, a few loose files are one part; only many of them are worth grouping.
389
+ if len(loose) >= (MIN_CLUSTER_FILES_BESIDE if subs else MIN_CLUSTER_FILES):
390
+ groups = _groups(m, part, base, loose)
391
+ if groups:
392
+ kids += groups
393
+ how = "folders and groups of files" if subs else "groups of files"
394
+ elif subs:
395
+ kids.append(Part(_dir_id(part.repo, base) + "#files", "files", "files in " + _under(part.path, base),
396
+ loose, base, part.module, part.repo, part.id))
397
+ else:
398
+ kids += [file_part(m, f, part.id) for f in loose]
399
+ elif subs and len(loose) > 1:
400
+ kids.append(Part(_dir_id(part.repo, base) + "#files", "files", "files in " + _under(part.path, base),
401
+ loose, base, part.module, part.repo, part.id))
402
+ else:
403
+ kids += [file_part(m, f, part.id) for f in loose]
404
+ if len(kids) == 1 and kids[0].kind in ("folder", "files") and set(kids[0].files) == set(part.files):
405
+ return split(m, kids[0])
406
+ return _pack(m, part, kids), how, shown_from
407
+
408
+
409
+ def _groups(m: Model, part: Part, base: str, loose: list) -> list:
410
+ """The loose files of a flat folder in groups (see the module's docstring); [] when they do not split cleanly."""
411
+ try:
412
+ import networkx as nx
413
+ except ImportError:
414
+ return []
415
+ U = {u for f in loose for u in m.units_of_file.get(f, ())}
416
+ if len(U) < 4:
417
+ return []
418
+ mod = part.module
419
+ stored = [s for s in m.systems.get(mod, []) if m.system_units.get(s)]
420
+ every = {u for f in m.files_of_module.get(mod, []) for u in m.units_of_file.get(f, ())}
421
+ found: list = [] # (units of U, stored system id or None, the code around it it uses)
422
+ if stored and base == m.modules[mod]["path"] and len(U) >= NEARLY_ALL * len(every):
423
+ found = [(set(m.system_units[s]) & U, s, []) for s in stored]
424
+ found = [x for x in found if len(x[0]) >= 2]
425
+ if not found:
426
+ found = _louvain(nx, m, U, set())
427
+ placed = set().union(*(x[0] for x in found)) if found else set()
428
+ if len(placed) < 0.5 * len(U):
429
+ # Few of them link to each other (a folder of tests, say): group them by the code they use instead, with
430
+ # that code in the graph and left out of the groups.
431
+ ctx = {v for u in U for v in m.uadj.get(u, ()) if v not in U}
432
+ again = _louvain(nx, m, U, ctx)
433
+ if again and len(set().union(*(x[0] for x in again))) > len(placed):
434
+ found = again
435
+ if not found:
436
+ return []
437
+ votes: dict = defaultdict(Counter)
438
+ for k, x in enumerate(found):
439
+ for u in x[0]:
440
+ votes[m.file_of.get(u) or u][k] += 1
441
+ by_group: dict = defaultdict(list)
442
+ left = []
443
+ for f in sorted(loose):
444
+ if votes.get(f):
445
+ by_group[sorted(votes[f].items(), key=lambda kv: (-kv[1], -len(found[kv[0]][0]), kv[0]))[0][0]].append(f)
446
+ else:
447
+ left.append(f)
448
+ if len(by_group) < 2 and not left:
449
+ return []
450
+ out, used = [], Counter()
451
+ folder = _dir_id(part.repo, base)
452
+ for k, fs in sorted(by_group.items()):
453
+ units, sid, around = found[k]
454
+ if len(fs) == 1: # a file and its own types: that is the file, not a group
455
+ out.append(file_part(m, fs[0], folder))
456
+ continue
457
+ # Named after its most linked type or file among the files it holds (a stored system keeps its id, but its
458
+ # anchor may sit in a file that went to another group).
459
+ own = {u for f in fs for u in m.units_of_file.get(f, ())} & units
460
+ anchor = sorted(own or units, key=lambda u: (-sum(w for v, w in m.uadj.get(u, {}).items() if v in own), u))[0]
461
+ nm = m.name.get(anchor) or m.files.get(anchor, {}).get("path", anchor).rsplit("/", 1)[-1]
462
+ label = f"{nm} group"
463
+ if sid is None:
464
+ used[nm] += 1
465
+ sid = f"{folder}#system:{nm}" + (f"~{used[nm]}" if used[nm] > 1 else "")
466
+ out.append(Part(sid, "system", label, fs, base, mod, part.repo, folder,
467
+ around=[m.name.get(v) or m.files.get(v, {}).get("path", v).rsplit("/", 1)[-1] for v in around]))
468
+ if len(left) == 1:
469
+ out.append(file_part(m, left[0], folder))
470
+ elif left:
471
+ out.append(Part(f"{folder}#other", "other", "other files in " + _under(part.path, base), left, base, mod,
472
+ part.repo, folder))
473
+ return out
474
+
475
+
476
+ def _louvain(nx, m: Model, U: set, ctx: set) -> list:
477
+ """Louvain communities of U (with ctx in the graph but not in the result) holding two or more units of U, as
478
+ [(units of U, None, the ctx units it is most linked to)]. [] when the split is weak."""
479
+ nodes = U | ctx
480
+ g = nx.Graph()
481
+ g.add_nodes_from(sorted(nodes))
482
+ for a in sorted(nodes):
483
+ for b, w in sorted(m.uadj.get(a, {}).items()):
484
+ if a < b and b in nodes and (a in U or b in U):
485
+ g.add_edge(a, b, weight=w)
486
+ if not g.number_of_edges():
487
+ return []
488
+ comms = nx.community.louvain_communities(g, weight="weight", seed=7)
489
+ if nx.community.modularity(g, comms, weight="weight") < MIN_MODULARITY:
490
+ return []
491
+ out = []
492
+ for c in sorted((c for c in comms if len(c & U) >= 2), key=lambda c: (-len(c & U), sorted(c & U)[0])):
493
+ around = sorted(c - U, key=lambda v: (-sum(g[v][w]["weight"] for w in g[v] if w in U), v))[:3]
494
+ out.append((c & U, None, around))
495
+ return out
496
+
497
+
498
+ # -- finding a part by its id ------------------------------------------------------------------------------------------
499
+ def find(m: Model, pid: Optional[str]) -> Optional[Part]:
500
+ """The part an id names: a module, a folder, a group, a file, a type, a repository, or a module's path."""
501
+ if not pid or not pid.strip():
502
+ if len(m.repos) == 1:
503
+ return Part(m.repos[0], "repo", m.repos[0], sorted(m.files), "", "", m.repos[0])
504
+ return Part("workspace", "workspace", "the workspace", sorted(m.files))
505
+ pid = pid.strip()
506
+ if pid in m.repos:
507
+ return Part(pid, "repo", pid, sorted(f for f, v in m.files.items() if v["repo"] == pid), "", "", pid)
508
+ if pid in m.modules:
509
+ return module_part(m, pid)
510
+ if pid in m.files:
511
+ return file_part(m, pid)
512
+ if pid in m.system_units:
513
+ mod = next((k for k, v in m.systems.items() if pid in v), "")
514
+ files = sorted({m.file_of.get(u) or u for u in m.system_units[pid]} & set(m.files))
515
+ return Part(pid, "system", m.system_name.get(pid, pid), files, m.modules.get(mod, {}).get("path", ""), mod,
516
+ m.modules.get(mod, {}).get("repo", ""), mod)
517
+ if m.kind.get(pid) == "type":
518
+ f = m.file_of.get(pid)
519
+ return Part(pid, "type", m.name[pid], [f] if f else [], m.files[f]["path"] if f else "",
520
+ m.files[f]["module"] if f else "", m.files[f]["repo"] if f else "", f or "", node=pid)
521
+ if pid.endswith("@more"):
522
+ owner = find(m, pid[:-len("@more")])
523
+ if owner is None:
524
+ return None
525
+ return next((k for k in split(m, owner)[0] if k.id == pid), None)
526
+ if ":dir:" in pid:
527
+ repo, _, rest = pid.partition(":dir:")
528
+ path, _, frag = rest.partition("#")
529
+ if frag == "files":
530
+ return folder_part(m, repo, path, loose_only=True)
531
+ folder = folder_part(m, repo, path)
532
+ if folder is None or not frag:
533
+ return folder
534
+ return _search(m, folder, pid)
535
+ # A path: a module's, or a folder's in one of the repositories.
536
+ path = pid.strip("/")
537
+ for k, v in sorted(m.modules.items()):
538
+ if v["path"] == path or (path in (".", "") and not v["path"]):
539
+ return module_part(m, k)
540
+ for r in m.repos:
541
+ p = folder_part(m, r, path)
542
+ if p is not None:
543
+ return p
544
+ f = next((i for i, v in m.files.items() if v["path"] == path), None)
545
+ return file_part(m, f) if f else None
546
+
547
+
548
+ def _search(m: Model, start: Part, pid: str, depth: int = 3) -> Optional[Part]:
549
+ """A part made by splitting `start` (a group, what no group took): looked for a few levels down."""
550
+ level = [start]
551
+ for _ in range(depth):
552
+ nxt = []
553
+ for p in level:
554
+ for k in split(m, p)[0]:
555
+ if k.id == pid:
556
+ return k
557
+ if k.kind in ("folder", "more", "module") and pid.startswith(k.id.split("@")[0].split("#")[0]):
558
+ nxt.append(k)
559
+ level = nxt
560
+ return None
561
+
562
+
563
+ # -- names -----------------------------------------------------------------------------------------------------------------
564
+ def ensure(con) -> None:
565
+ con.execute("CREATE TABLE IF NOT EXISTS part_names (part_id TEXT PRIMARY KEY, name TEXT NOT NULL, summary TEXT,"
566
+ " layer TEXT NOT NULL, source TEXT, kind TEXT, parent TEXT, members TEXT, evidence TEXT, created TEXT)")
567
+
568
+
569
+ def _rows(con) -> dict:
570
+ try:
571
+ return {r[0]: {"id": r[0], "name": r[1], "summary": r[2] or "", "layer": r[3], "kind": r[4], "parent": r[5],
572
+ "members": json.loads(r[6] or "[]"), "evidence": json.loads(r[7] or "[]")}
573
+ for r in con.execute("SELECT part_id, name, summary, layer, kind, parent, members, evidence FROM part_names")}
574
+ except Exception: # a store with no names yet
575
+ return {}
576
+
577
+
578
+ def names(con, parts: list) -> dict:
579
+ """{part id: {"name", "summary", "by"?, "stale"?, "from"?}} for the parts that have a name, or inherit one."""
580
+ rows = _rows(con)
581
+ if not rows:
582
+ return {}
583
+ out = {}
584
+ ids = {p.id for p in parts}
585
+ for p in parts:
586
+ r = rows.get(p.id)
587
+ if r is None and p.kind == "system":
588
+ # Its anchor (and so its id) changed: the name of the old group of this folder it shares most files with.
589
+ best, score = None, 0.0
590
+ for x in rows.values():
591
+ if x["kind"] == "system" and x["parent"] == p.parent and x["id"] not in ids and x["members"]:
592
+ a, b = set(x["members"]), set(p.files)
593
+ j = len(a & b) / len(a | b)
594
+ if j > score:
595
+ best, score = x, j
596
+ if best is not None and score >= KEPT_FOR_FRESH:
597
+ r = {**best, "from": best["id"]}
598
+ if r is None:
599
+ continue
600
+ item = {"name": r["name"], "summary": r["summary"]}
601
+ if r["layer"] == "intent":
602
+ item["by"] = "the person"
603
+ if r.get("from"):
604
+ item["from"] = r["from"]
605
+ stale = _stale(con, p, r)
606
+ if stale:
607
+ item["stale"] = stale
608
+ out[p.id] = item
609
+ return out
610
+
611
+
612
+ def _stale(con, p: Part, r: dict) -> Optional[str]:
613
+ if p.kind not in ("folder", "module", "repo", "file", "type") and r["members"]:
614
+ had = set(r["members"])
615
+ kept = len(had & set(p.files)) / len(had)
616
+ if kept < KEPT_FOR_FRESH or len(set(p.files) - had) > len(had):
617
+ return (f"may be stale: it kept {round(100 * kept)}% of the {len(had)} files it had when named, and has"
618
+ f" {len(p.files)} now")
619
+ ev = r["evidence"]
620
+ if ev:
621
+ gone = [e for e in ev if con.execute("SELECT 1 FROM nodes WHERE id = ?", (e,)).fetchone() is None]
622
+ if len(gone) * 2 > len(ev):
623
+ return "may be stale: most of the code it was named from is gone"
624
+ return None
625
+
626
+
627
+ def name_part(con, part_id: str, name: str, summary: str = "", evidence: Optional[list] = None,
628
+ layer: str = "inferred", source: str = "mcp") -> dict:
629
+ """Store a name and a one-line summary for a part, replacing an earlier one."""
630
+ name, summary = (name or "").strip(), " ".join((summary or "").split())
631
+ if not name:
632
+ return {"error": "name is empty: give the part a short name, such as \"Validation rules\"."}
633
+ if layer not in ("inferred", "intent"):
634
+ return {"error": "layer must be inferred (your reading of the code) or intent (the person said it)"}
635
+ m = model(con)
636
+ p = find(m, part_id)
637
+ if p is None:
638
+ return {"error": f"no part {part_id!r}. Part ids come from module_outline (`leyline outline`)."}
639
+ if p.kind in ("more", "workspace"):
640
+ return {"error": f"{part_id!r} only holds the rest of a level; name the parts inside it."}
641
+ evidence = [e for e in (evidence or []) if e]
642
+ if layer == "inferred":
643
+ if not evidence:
644
+ return {"error": "an inferred name needs evidence: the node ids of the code you read to name it"
645
+ " (from the outline's busiest and key items, or `search`)."}
646
+ missing = [e for e in evidence if con.execute("SELECT 1 FROM nodes WHERE id = ?", (e,)).fetchone() is None]
647
+ if missing:
648
+ return {"error": f"evidence ids not on the map: {missing[:3]}. Use node ids from module_outline or `search`."}
649
+ ensure(con)
650
+ with con:
651
+ con.execute("INSERT OR REPLACE INTO part_names VALUES (?,?,?,?,?,?,?,?,?,?)",
652
+ (p.id, name, summary, layer, source, p.kind, p.parent, json.dumps(sorted(p.files)),
653
+ json.dumps(evidence), time.strftime("%Y-%m-%dT%H:%M:%S")))
654
+ if p.id in m.system_units: # a system the map proposed: `overview` and the map's system boxes show it too
655
+ store.annotate(con, p.id, "name", name, layer, source, None, evidence)
656
+ if summary:
657
+ store.annotate(con, p.id, "responsibility", summary, layer, source, None, evidence)
658
+ return {"part_id": p.id, "name": name, "summary": summary, "layer": layer, "files": len(p.files),
659
+ "next": "Name the other parts the same way; module_outline shows the names, and the map page after `map`."}
660
+
661
+
662
+ def labels(con, node_ids: list) -> dict:
663
+ """{node id: the name of the most specific named part it sits in} for the ids that sit in one."""
664
+ rows = _rows(con)
665
+ if not rows or not node_ids:
666
+ return {}
667
+ m = model(con)
668
+ out = {}
669
+ for nid in node_ids:
670
+ f = m._file(nid)
671
+ if f is None:
672
+ continue
673
+ info = m.files[f]
674
+ best = None # (specificity, name): fewer files is more specific
675
+ for r in rows.values():
676
+ pid = r["id"]
677
+ if pid == f or (r["kind"] not in ("folder", "module", "repo") and f in r["members"]):
678
+ size = len(r["members"]) or 1
679
+ elif r["kind"] == "folder" and pid.startswith(info["repo"] + ":dir:") and \
680
+ info["path"].startswith(pid.split(":dir:", 1)[1].rstrip("/") + "/"):
681
+ size = 10_000 - pid.count("/")
682
+ elif r["kind"] == "module" and pid == info["module"]:
683
+ size = 100_000
684
+ else:
685
+ continue
686
+ if best is None or size < best[0]:
687
+ best = (size, r["name"])
688
+ if best:
689
+ out[nid] = best[1]
690
+ return out
691
+
692
+
693
+ def named_parts(con) -> dict:
694
+ """{part id: {"id", "name", "summary", "kind", "module"}} for every part given a name, for `overview`."""
695
+ out = {}
696
+ for k, r in sorted(_rows(con).items()):
697
+ mod = k if r["kind"] == "module" else ""
698
+ if not mod and r["members"]:
699
+ row = con.execute("SELECT parent_id FROM nodes WHERE id = ?", (r["members"][0],)).fetchone()
700
+ mod = row[0] if row else ""
701
+ out[k] = {"id": k, "name": r["name"], "summary": r["summary"], "kind": r["kind"], "module": mod or ""}
702
+ return out
703
+
704
+
705
+ # -- describing a part --------------------------------------------------------------------------------------------------
706
+ def _outer(m: Model, g: str, within: Part) -> tuple[str, str]:
707
+ """(id, label) for a file outside `within`: the folder next to it in the same module, or its module."""
708
+ info = m.files[g]
709
+ if within.kind in ("workspace", "repo") or info["module"] != within.module:
710
+ mod = info["module"]
711
+ if mod is None:
712
+ return info["repo"], info["repo"]
713
+ lab = m.modules[mod]["path"] or "(repository root)"
714
+ return mod, (lab if info["repo"] == within.repo or not within.repo else f"{info['repo']}/{lab}")
715
+ folder = posixpath.dirname(info["path"])
716
+ base = within.path
717
+ common = posixpath.commonpath([base, folder]) if base and folder else ""
718
+ rest = info["path"][len(common) + 1:] if common else info["path"]
719
+ if "/" not in rest:
720
+ return _dir_id(info["repo"], common) + "#files", "files in " + _folder_label(m, within.module, common)
721
+ sub = (common + "/" if common else "") + rest.split("/", 1)[0]
722
+ return _dir_id(info["repo"], sub), _folder_label(m, within.module, sub)
723
+
724
+
725
+ def _entries(m: Model, fns: list, tests: int) -> dict:
726
+ kinds = Counter(m.entry_kind[f] for f in fns if f in m.entry_kind)
727
+ out: dict = {}
728
+ if kinds:
729
+ out["kinds"] = dict(kinds.most_common())
730
+ shown = [f for f in fns if m.entry_kind.get(f) in ("route", "command", "entry", "message")]
731
+ shown.sort(key=lambda f: (-m.flows.get(f, 0), f))
732
+ if shown:
733
+ out["examples"] = [{"id": f, "name": m.route.get(f) or (m.files[m.file_of[f]]["path"] if m.name.get(f) in TOP
734
+ else _qual(m, f)), "kind": m.entry_kind[f]}
735
+ for f in shown[:3]]
736
+ out["tests"] = tests
737
+ return out
738
+
739
+
740
+ def _entry_text(e: dict) -> str:
741
+ bits = [f"{n:,} {ENTRY_WORDS[k][n != 1]}" for k, n in e.get("kinds", {}).items()]
742
+ if e.get("tests"):
743
+ bits.append(f"{e['tests']:,} test{'s' if e['tests'] != 1 else ''}")
744
+ return ", ".join(bits) or "none"
745
+
746
+
747
+ def describe(con, m: Model, p: Part, owner: dict, within: Part, given: dict, depth: int) -> dict:
748
+ """One part as the outline shows it. `owner` maps each file of the level to its sibling part (id, label)."""
749
+ files = set(p.files)
750
+ fns = [f for x in p.files for f in m.functions.get(x, ())]
751
+ tests = sum(m.tests.get(x, 0) for x in p.files)
752
+ g = given.get(p.id)
753
+ out: dict = {"id": p.id, "kind": p.kind, "name": g["name"] if g else p.label}
754
+ if g:
755
+ out["named"] = True
756
+ if g["name"] != p.label:
757
+ out["label"] = p.label
758
+ if g.get("summary"):
759
+ out["summary"] = g["summary"]
760
+ for k, to in (("by", "named_by"), ("from", "named_as"), ("stale", "stale")):
761
+ if g.get(k):
762
+ out[to] = g[k]
763
+ if p.path and p.kind in ("folder", "files", "system", "other", "module"):
764
+ out["path"] = p.path + "/"
765
+ out["size"] = {"files": len(files), "functions": len(fns), "lines": m.lines(files)}
766
+ out["entry_points"] = _entries(m, fns, tests)
767
+ if fns:
768
+ out["on_a_test_path"] = f"{sum(1 for f in fns if f in m.tested):,} of {len(fns):,} functions"
769
+ uses, used_by = Counter(), Counter()
770
+ label: dict = {}
771
+ for x in files:
772
+ for side, acc in ((m.fout.get(x, {}), uses), (m.fin.get(x, {}), used_by)):
773
+ for y, n in side.items():
774
+ if y in files:
775
+ continue
776
+ k, lab = owner.get(y) or _outer(m, y, within)
777
+ label[k] = lab
778
+ acc[k] += n
779
+ for k in label: # a part elsewhere that has a name is called by it
780
+ if k in given:
781
+ label[k] = given[k]["name"]
782
+ out["uses"] = [{"id": k, "name": label[k], "links": n} for k, n in uses.most_common(6)]
783
+ out["used_by"] = [{"id": k, "name": label[k], "links": n} for k, n in used_by.most_common(6)]
784
+ if len(uses) > 6 or len(used_by) > 6:
785
+ out["links_total"] = {"uses": sum(uses.values()), "used_by": sum(used_by.values())}
786
+ chans: dict = {}
787
+ for fs, fd, ch, addr in {c for x in files for c in m.channels.get(x, ())}:
788
+ a, b = fs in files, fd in files
789
+ if a == b:
790
+ continue
791
+ other = fd if a else fs
792
+ k, lab = owner.get(other) or _outer(m, other, within)
793
+ c = chans.setdefault((ch, "to" if a else "from", k), {"channel": ch, "direction": "to" if a else "from",
794
+ "other": given.get(k, {}).get("name", lab), "links": 0,
795
+ "example": addr})
796
+ c["links"] += 1
797
+ if chans:
798
+ out["channels"] = sorted(chans.values(), key=lambda c: (-c["links"], c["channel"], c["other"]))[:6]
799
+ busy = sorted((f for f in fns if m.flows.get(f)), key=lambda f: (-m.flows[f], f))[:4]
800
+ if busy:
801
+ out["busiest"] = [{"id": f, "name": _qual(m, f), "flows": m.flows[f]} for f in busy]
802
+ # The types other code leans on most, one of each name (every component has its own Props).
803
+ types, seen = [], set()
804
+ for u in sorted({u for x in p.files for u in m.units_of_file.get(x, ()) if m.kind.get(u) == "type"},
805
+ key=lambda u: (-sum(m.uadj.get(u, {}).values()), -m.lines_of.get(u, 0), u)):
806
+ if m.name[u] not in seen and len(types) < 4:
807
+ seen.add(m.name[u])
808
+ types.append(u)
809
+ if types:
810
+ out["key_types"] = [{"id": t, "name": m.name[t]} for t in types]
811
+ if p.around:
812
+ out["around"] = p.around
813
+ if p.kind == "more":
814
+ out["holds"] = [k.label for k in p.rest]
815
+ if depth > 1:
816
+ kids, _, _ = split(m, p)
817
+ if kids:
818
+ kg = names(con, kids)
819
+ out["parts"] = [{"id": k.id, "name": kg[k.id]["name"] if k.id in kg else k.label, "files": len(k.files)}
820
+ for k in kids]
821
+ if depth > 2:
822
+ for item, k in zip(out["parts"], kids):
823
+ sub, _, _ = split(m, k)
824
+ if sub:
825
+ sg = names(con, sub)
826
+ item["parts"] = [{"id": s.id, "name": sg[s.id]["name"] if s.id in sg else s.label,
827
+ "files": len(s.files)} for s in sub]
828
+ out["drill"] = f'module_outline("{p.id}")'
829
+ return out
830
+
831
+
832
+ def _qual(m: Model, f: str) -> str:
833
+ if m.name.get(f) in TOP and f in m.file_of:
834
+ return "top level of " + m.files[m.file_of[f]]["path"].rsplit("/", 1)[-1]
835
+ u = m.unit.get(f)
836
+ if u and u != f and m.kind.get(u) == "type" and m.name.get(f) != m.name.get(u):
837
+ return f"{m.name[u]}.{m.name.get(f, f)}"
838
+ return m.name.get(f, f)
839
+
840
+
841
+ def _risks(parts: list) -> Optional[str]:
842
+ """What makes a part risky beside its siblings: the busiest, the most depended on, those few tests reach."""
843
+ if len(parts) < 3:
844
+ return None
845
+ load = [sum(b["flows"] for b in p.get("busiest", ())) for p in parts]
846
+ deps = [sum(u["links"] for u in p.get("used_by", ())) for p in parts]
847
+ for p, l, d in zip(parts, load, deps):
848
+ r = []
849
+ if l and l == max(load):
850
+ r.append("busy: its functions are on the most flows")
851
+ if d and d == max(deps):
852
+ r.append("many dependents: the most links into it")
853
+ fn = p["size"]["functions"]
854
+ if fn >= 5:
855
+ on = int(p["on_a_test_path"].split(" of ")[0].replace(",", ""))
856
+ if on < 0.3 * fn:
857
+ r.append(f"few tests: {on} of {fn} functions are on a test's path")
858
+ if r:
859
+ p["risky"] = r
860
+ few = [p for p in parts if any(x.startswith("few tests") for x in p.get("risky", ()))]
861
+ if len(few) >= 0.8 * len(parts):
862
+ for p in few: # said once for the level rather than on every part
863
+ p["risky"] = [x for x in p["risky"] if not x.startswith("few tests")]
864
+ if not p["risky"]:
865
+ del p["risky"]
866
+ return "Few tests reach any of these parts: most of their functions are on no test's path."
867
+ return None
868
+
869
+
870
+ def _leaf(con, m: Model, p: Part, given: dict) -> dict:
871
+ """A file, a type or a group with one file: its key types and functions, most flows through them first."""
872
+ within = Part(p.id, p.kind, p.label, p.files, posixpath.dirname(p.path) if p.kind in ("file", "type") else p.path,
873
+ p.module, p.repo)
874
+ out = describe(con, m, p, {}, within, given, 1)
875
+ out.pop("drill", None)
876
+ if p.kind == "type":
877
+ members = [i for i in m.functions.get(p.files[0], ()) if m.unit.get(i) == p.node] if p.files else []
878
+ else:
879
+ members = [i for f in p.files for i in m.functions.get(f, ())]
880
+ key = sorted(members, key=lambda i: (-m.flows.get(i, 0), -m.lines_of.get(i, 0), i))[:12]
881
+ out["key"] = [{"id": i, "name": _qual(m, i), "flows": m.flows.get(i, 0), "lines": m.lines_of.get(i, 0),
882
+ **({"entry": m.entry_kind[i]} if i in m.entry_kind else {}),
883
+ **({"tested": True} if i in m.tested else {})} for i in key]
884
+ if len(p.files) > 1:
885
+ out["files"] = {"total": len(p.files), "items": [m.files[f]["path"] for f in p.files[:15]]}
886
+ return out
887
+
888
+
889
+ def outline(con, part_id: Optional[str] = None, depth: int = 2) -> dict:
890
+ """One level of a module (or of the repository, with no part): its parts, described."""
891
+ depth = max(1, min(int(depth or 2), 3))
892
+ m = model(con)
893
+ p = find(m, part_id)
894
+ if p is None:
895
+ return {"error": f"no module or part {part_id!r} on the map. Call module_outline() with no module to see the"
896
+ " modules, or give a module's path (such as editor/core) or a part id from an outline."}
897
+ given = names(con, [p])
898
+ kids, how, shown_from = split(m, p)
899
+ head = {"id": p.id, "kind": p.kind, "name": given[p.id]["name"] if p.id in given else p.label}
900
+ if p.id in given:
901
+ head["named"] = True
902
+ if given[p.id].get("summary"):
903
+ head["summary"] = given[p.id]["summary"]
904
+ if given[p.id].get("stale"):
905
+ head["stale"] = given[p.id]["stale"]
906
+ if p.path:
907
+ head["path"] = p.path + ("/" if p.kind not in ("file", "type") else "")
908
+ fns = sum(len(m.functions.get(f, ())) for f in p.files)
909
+ head["size"] = {"files": len(p.files), "functions": fns, "lines": m.lines(p.files)}
910
+ if not kids:
911
+ out = {**head, **{k: v for k, v in _leaf(con, m, p, given).items() if k not in head}}
912
+ out["next"] = ("Read the key code (`source` with an id from key) before saying what it does; name this part"
913
+ f" with name_part(\"{p.id}\", name, summary, evidence).")
914
+ return out
915
+ owner = {}
916
+ for k in kids:
917
+ for x in (k.rest if k.kind == "more" else [k]): # past the first page, each part is still itself
918
+ for f in x.files:
919
+ owner[f] = (x.id, x.label)
920
+ given = {**{r["id"]: {"name": r["name"]} for r in _rows(con).values()}, **names(con, kids), **given}
921
+ for f, (k, lab) in list(owner.items()):
922
+ if k in given:
923
+ owner[f] = (k, given[k]["name"])
924
+ within = Part(p.id, p.kind, p.label, p.files, shown_from or p.path, p.module, p.repo)
925
+ parts = [describe(con, m, k, owner, within, given, depth) for k in kids]
926
+ note = _risks(parts)
927
+ named = sum(1 for x in parts if x.get("named"))
928
+ out = {**head, "split": how}
929
+ if shown_from:
930
+ out["shown_from"] = shown_from + "/"
931
+ out["names"] = f"{named} of {len(parts)} parts {'has' if named == 1 else 'have'} a name"
932
+ if note:
933
+ out["note"] = note
934
+ out["parts"] = parts
935
+ out["next"] = ("Drill into a part with module_outline(\"<part id>\") (CLI: leyline outline <part id>). Read a part's"
936
+ " key code (`source`, `context`), then name it with name_part(part_id, name, summary, evidence).")
937
+ return out
938
+
939
+
940
+ # -- text for the command line ----------------------------------------------------------------------------------------
941
+ def _n(n: int, word: str) -> str:
942
+ return f"{n:,} {word}{'' if n == 1 else 's'}"
943
+
944
+
945
+ def _size(s: dict) -> str:
946
+ return (f"{s['files']:,} file{'s' if s['files'] != 1 else ''}, {s['functions']:,} function"
947
+ f"{'s' if s['functions'] != 1 else ''}, {s['lines']:,} line{'s' if s['lines'] != 1 else ''}")
948
+
949
+
950
+ def _links(xs: list) -> str:
951
+ return ", ".join(f"{x['name']} ({x['links']:,})" for x in xs)
952
+
953
+
954
+ def _part_lines(p: dict, n: Optional[int], pad: str = " ") -> list:
955
+ name = p["name"] + (f" ({p['label']})" if p.get("label") else "")
956
+ tag = f" [{p['kind']}]" if p["kind"] not in ("folder", "module", "repo") else ""
957
+ L = [(f"{n}. " if n else "") + f"{name}{tag}: {_size(p['size'])}", f"{pad}id: {p['id']}"]
958
+ if p.get("summary"):
959
+ L.append(f"{pad}{p['summary']}")
960
+ if p.get("stale"):
961
+ L.append(f"{pad}name {p['stale']}")
962
+ if p.get("named_as"):
963
+ L.append(f"{pad}named as {p['named_as']}, which shared most of its files")
964
+ e = p.get("entry_points") or {}
965
+ if e:
966
+ L.append(f"{pad}entry points: {_entry_text(e)}" + (f"; {p['on_a_test_path']} on a test's path"
967
+ if p.get("on_a_test_path") else ""))
968
+ if e.get("examples"):
969
+ L.append(f"{pad} e.g. " + ", ".join(x["name"] for x in e["examples"]))
970
+ if p.get("around"):
971
+ L.append(f"{pad}around: {', '.join(p['around'])}")
972
+ if p.get("uses"):
973
+ L.append(f"{pad}uses: {_links(p['uses'])}")
974
+ if p.get("used_by"):
975
+ L.append(f"{pad}used by: {_links(p['used_by'])}")
976
+ for c in p.get("channels", ()):
977
+ L.append(f"{pad}channel: {c['channel']} {c['direction']} {c['other']}, {c['links']} link{'s' if c['links'] != 1 else ''}"
978
+ + (f" (e.g. {c['example']})" if c.get("example") else ""))
979
+ if p.get("busiest"):
980
+ L.append(f"{pad}busiest: " + ", ".join(f"{b['name']} ({_n(b['flows'], 'flow')})" for b in p["busiest"][:3]))
981
+ if p.get("key_types"):
982
+ L.append(f"{pad}key types: " + ", ".join(t["name"] for t in p["key_types"]))
983
+ if p.get("risky"):
984
+ L.append(f"{pad}risky: " + "; ".join(p["risky"]))
985
+ if p.get("holds"):
986
+ L.append(f"{pad}holds: " + ", ".join(p["holds"][:10]) + (" and more" if len(p["holds"]) > 10 else ""))
987
+ if p.get("parts"):
988
+ L.append(f"{pad}parts: " + ", ".join(f"{x['name']} ({x['files']} file{'s' if x['files'] != 1 else ''})"
989
+ for x in p["parts"][:8])
990
+ + (f" and {len(p['parts']) - 8} more" if len(p["parts"]) > 8 else ""))
991
+ return L
992
+
993
+
994
+ def text(r: dict) -> str:
995
+ if "error" in r:
996
+ return r["error"]
997
+ L = [f"{r['name']} ({r['kind']}" + (f", {r['path']}" if r.get("path") and r["path"] != r["name"] else "")
998
+ + f"): {_size(r['size'])}."]
999
+ if r.get("summary"):
1000
+ L.append(r["summary"])
1001
+ if r.get("stale"):
1002
+ L.append(f"Its name {r['stale']}.")
1003
+ if "parts" in r and "split" in r:
1004
+ L.append(f"Split by {r['split']}" + (f" (shown from {r['shown_from']}, the folder that holds nearly all of it)"
1005
+ if r.get("shown_from") else "") + f". {r['names']}."
1006
+ + (f" {r['note']}" if r.get("note") else ""))
1007
+ L.append("")
1008
+ for k, p in enumerate(r["parts"], 1):
1009
+ L += _part_lines(p, k)
1010
+ else:
1011
+ L += _part_lines({**r, "name": "This part"}, None, "")[2:]
1012
+ if r.get("files"):
1013
+ f = r["files"]
1014
+ L.append("Files: " + ", ".join(x.rsplit("/", 1)[-1] for x in f["items"])
1015
+ + (f" and {f['total'] - len(f['items'])} more" if f["total"] > len(f["items"]) else ""))
1016
+ if r.get("key"):
1017
+ L.append("Key types and functions:")
1018
+ for x in r["key"]:
1019
+ L.append(f" {x['name']} {_n(x['flows'], 'flow')}, {_n(x['lines'], 'line')}" + (f", {x['entry']}" if x.get("entry") else "")
1020
+ + ("" if x.get("tested") else ", on no test's path") + f" [{x['id']}]")
1021
+ name = f"`leyline name-part {r['id'] if 'key' in r else '<part id>'} \"<name>\" --summary \"<one line>\" --evidence <node ids>`"
1022
+ if "key" in r:
1023
+ L += ["", f"Read the key code (`leyline source <id>`) before saying what it does; then name this part: {name}."]
1024
+ else:
1025
+ L += ["", "Drill into a part: `leyline outline <part id>`. Read a part's key code (`leyline source <id>`, `leyline"
1026
+ f" context <name>`; `--json` gives the ids), then name it: {name}."]
1027
+ return "\n".join(L)
1028
+
1029
+
1030
+ # -- for the map page ---------------------------------------------------------------------------------------------------
1031
+ MAP_MIN_FILES = 30 # a module this large is drawn as its parts
1032
+ MAP_MOST = 40 # the largest this many such modules
1033
+
1034
+
1035
+ def map_parts(con) -> list[dict]:
1036
+ """For each large module: its parts two levels down, each with its files, name and size, for export.py."""
1037
+ m = model(con)
1038
+ big = sorted((k for k in m.modules if len(m.files_of_module.get(k, ())) >= MAP_MIN_FILES),
1039
+ key=lambda k: (-len(m.files_of_module[k]), k))[:MAP_MOST]
1040
+ out = []
1041
+ for mod in big:
1042
+ kids, _, _ = split(m, module_part(m, mod))
1043
+ if len(kids) < 2:
1044
+ continue
1045
+ given = names(con, kids)
1046
+
1047
+ def item(k, g):
1048
+ fns = sum(len(m.functions.get(f, ())) for f in k.files)
1049
+ x = {"id": k.id, "kind": k.kind, "name": g[k.id]["name"] if k.id in g else k.label, "label": k.label,
1050
+ "files": k.files, "functions": fns, "lines": m.lines(k.files)}
1051
+ if k.id in g:
1052
+ x["named"] = True
1053
+ for key in ("summary", "stale"):
1054
+ if g[k.id].get(key):
1055
+ x[key] = g[k.id][key]
1056
+ return x
1057
+ parts = []
1058
+ for k in kids:
1059
+ x = item(k, given)
1060
+ sub, _, _ = split(m, k)
1061
+ if len(sub) >= 2 and len(k.files) >= 2:
1062
+ sg = names(con, sub)
1063
+ x["parts"] = [item(s, sg) for s in sub]
1064
+ parts.append(x)
1065
+ out.append({"module": mod, "parts": parts})
1066
+ return out