edify-agents-cli 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edify/__init__.py +19 -0
- edify/__main__.py +6 -0
- edify/_toml.py +286 -0
- edify/agents.py +261 -0
- edify/anim.py +499 -0
- edify/assets/commands/build.md +64 -0
- edify/assets/commands/plan.md +68 -0
- edify/assets/commands/spec.md +63 -0
- edify/assets/commands/tasks.md +65 -0
- edify/assets/commands/verify.md +67 -0
- edify/assets/formats/plan.example.md +455 -0
- edify/assets/formats/plan.format.md +128 -0
- edify/assets/formats/spec.example.md +483 -0
- edify/assets/formats/spec.format.md +116 -0
- edify/assets/formats/tasks.example.md +712 -0
- edify/assets/formats/tasks.format.md +199 -0
- edify/assets/harvest/harvest.md +91 -0
- edify/assets/harvest/sources.md +295 -0
- edify/assets/harvest/vocab.md +52 -0
- edify/assets/manifest.md +30 -0
- edify/assets/mcp/default-registry.md +66 -0
- edify/assets/skills/adversary.md +61 -0
- edify/assets/skills/debugger.md +64 -0
- edify/assets/skills/implementer-general.md +53 -0
- edify/assets/skills/implementer-go.md +63 -0
- edify/assets/skills/implementer-node.md +61 -0
- edify/assets/skills/implementer-postgres.md +63 -0
- edify/assets/skills/implementer-python.md +65 -0
- edify/assets/skills/implementer-react.md +65 -0
- edify/assets/skills/planner.md +58 -0
- edify/assets/skills/test-writer-node.md +64 -0
- edify/assets/skills/test-writer-python.md +60 -0
- edify/assets/skills/verifier.md +67 -0
- edify/assets/templates/CLAUDE.md +18 -0
- edify/assets.py +77 -0
- edify/cli.py +681 -0
- edify/commands/__init__.py +1 -0
- edify/commands/check_cmd.py +72 -0
- edify/commands/doctor_cmd.py +206 -0
- edify/commands/feedback_cmd.py +90 -0
- edify/commands/governance_cmd.py +65 -0
- edify/commands/graph_cmd.py +156 -0
- edify/commands/harvest_cmd.py +538 -0
- edify/commands/init_cmd.py +333 -0
- edify/commands/init_new_cmd.py +151 -0
- edify/commands/license_cmd.py +80 -0
- edify/commands/mcp_cmd.py +159 -0
- edify/commands/self_cmd.py +185 -0
- edify/commands/setup_cmd.py +79 -0
- edify/commands/skills_cmd.py +177 -0
- edify/commands/upgrade_cmd.py +178 -0
- edify/context.py +57 -0
- edify/conventions.py +275 -0
- edify/detect.py +313 -0
- edify/distribution.py +40 -0
- edify/errors.py +66 -0
- edify/feedback.py +207 -0
- edify/formats/__init__.py +11 -0
- edify/formats/check.py +573 -0
- edify/formats/document.py +238 -0
- edify/formats/finding.py +50 -0
- edify/formats/fix.py +114 -0
- edify/formats/mcp.py +126 -0
- edify/formats/plan.py +151 -0
- edify/formats/skill.py +139 -0
- edify/formats/spec.py +164 -0
- edify/formats/tasks.py +321 -0
- edify/governance.py +316 -0
- edify/graph/__init__.py +11 -0
- edify/graph/extract.py +351 -0
- edify/graph/ignore.py +107 -0
- edify/graph/langs/__init__.py +93 -0
- edify/graph/langs/ctags.py +145 -0
- edify/graph/langs/markdown.py +41 -0
- edify/graph/langs/patterns.py +361 -0
- edify/graph/langs/python_ast.py +154 -0
- edify/graph/langs/sqlish.py +86 -0
- edify/graph/model.py +102 -0
- edify/graph/queries.py +211 -0
- edify/graph/store.py +157 -0
- edify/harvest/__init__.py +28 -0
- edify/harvest/ledger.py +191 -0
- edify/harvest/screen.py +236 -0
- edify/harvest/sources.py +298 -0
- edify/host.py +345 -0
- edify/interview.py +354 -0
- edify/library.py +176 -0
- edify/licensing/__init__.py +30 -0
- edify/licensing/ed25519.py +154 -0
- edify/licensing/projects.py +223 -0
- edify/licensing/tier.py +172 -0
- edify/licensing/token.py +149 -0
- edify/paths.py +282 -0
- edify/selfinstall.py +244 -0
- edify/skills.py +108 -0
- edify/tsv.py +109 -0
- edify/ui.py +334 -0
- edify_agents_cli-0.2.0.dist-info/METADATA +408 -0
- edify_agents_cli-0.2.0.dist-info/RECORD +102 -0
- edify_agents_cli-0.2.0.dist-info/WHEEL +4 -0
- edify_agents_cli-0.2.0.dist-info/entry_points.txt +2 -0
- edify_agents_cli-0.2.0.dist-info/licenses/LICENSE +21 -0
edify/governance.py
ADDED
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
"""The ledger: every file EDIFY put in this repository, and where it came from.
|
|
2
|
+
|
|
3
|
+
A skill file has carried its own provenance since the first release. Nothing else
|
|
4
|
+
did — a command file, a format example, the seeded registry and the scraped
|
|
5
|
+
conventions all arrived with no record of what installed them, from which version,
|
|
6
|
+
or whether anybody has changed them since. That is a gap in exactly the thing
|
|
7
|
+
governance is about: a control counts as governance if a person
|
|
8
|
+
can look at it and tell whether the work complied.
|
|
9
|
+
|
|
10
|
+
So every file EDIFY writes is recorded in `.edify/governance.tsv`, one row each:
|
|
11
|
+
where it is, what it is, where it came from, under what licence, and its hash at the
|
|
12
|
+
moment it was installed. `edify governance verify` reads that back against the disk.
|
|
13
|
+
|
|
14
|
+
It is a ledger, not a lock. Nothing here prevents an edit — it makes one visible,
|
|
15
|
+
which is the whole claim. Eight origins, and `verify` treats them differently
|
|
16
|
+
because they are owned by different people:
|
|
17
|
+
|
|
18
|
+
library a curated skill entry — EDIFY's to replace, so an edit is a warning
|
|
19
|
+
shipped a command or format file — same
|
|
20
|
+
seeded the MCP registry, written once and then the repository's to maintain
|
|
21
|
+
scraped conventions, derived from this repository's own config files
|
|
22
|
+
generated a host command pointer
|
|
23
|
+
merged CLAUDE.md, AGENTS.md and the rest, where EDIFY owns a block and
|
|
24
|
+
somebody else owns the file
|
|
25
|
+
answered .edify/profile.md — what a person said when `init new` asked
|
|
26
|
+
team an entry a team admitted — written only by the team edition
|
|
27
|
+
|
|
28
|
+
The graph is deliberately not in here. It is regenerated output, it changes on every
|
|
29
|
+
build, and it already carries its own checksums in `.edify/graph/meta`.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
from dataclasses import dataclass
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
|
|
37
|
+
from . import __version__
|
|
38
|
+
from .distribution import DIST_NAME
|
|
39
|
+
from .errors import EdifyError
|
|
40
|
+
from .formats import skill as skill_format
|
|
41
|
+
from .host import written_by_edify
|
|
42
|
+
from .paths import Layout
|
|
43
|
+
from .tsv import read_rows, sha256_file, write_rows
|
|
44
|
+
|
|
45
|
+
LEDGER_HEADER = ("path", "kind", "origin", "provenance", "license", "source", "sha256", "edify")
|
|
46
|
+
|
|
47
|
+
# The licence the shipped methodology tree is under: the same licence `LICENSE` and
|
|
48
|
+
# the README state, so the ledger, the file and the page cannot drift apart. Skill
|
|
49
|
+
# entries override it from their own frontmatter, which is how an adapted entry
|
|
50
|
+
# stays governed by its upstream.
|
|
51
|
+
SHIPPED_LICENSE = "MIT"
|
|
52
|
+
|
|
53
|
+
ORIGINS = ("library", "shipped", "seeded", "scraped", "generated", "merged", "answered", "team")
|
|
54
|
+
|
|
55
|
+
# An edit to one of these is EDIFY's business: `upgrade` will overwrite it.
|
|
56
|
+
EDIFY_OWNED = ("library", "shipped", "generated")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True)
|
|
60
|
+
class Record:
|
|
61
|
+
path: str
|
|
62
|
+
kind: str
|
|
63
|
+
origin: str
|
|
64
|
+
provenance: str
|
|
65
|
+
license: str
|
|
66
|
+
source: str
|
|
67
|
+
sha256: str
|
|
68
|
+
edify: str
|
|
69
|
+
|
|
70
|
+
def row(self) -> list[str]:
|
|
71
|
+
return [
|
|
72
|
+
self.path,
|
|
73
|
+
self.kind,
|
|
74
|
+
self.origin,
|
|
75
|
+
self.provenance,
|
|
76
|
+
self.license,
|
|
77
|
+
self.source,
|
|
78
|
+
self.sha256,
|
|
79
|
+
self.edify,
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
def as_dict(self) -> dict[str, str]:
|
|
83
|
+
return dict(zip(LEDGER_HEADER, self.row()))
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(frozen=True)
|
|
87
|
+
class Problem:
|
|
88
|
+
path: str
|
|
89
|
+
state: str # missing | modified | edited | unrecorded | unknown-origin
|
|
90
|
+
detail: str
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def serious(self) -> bool:
|
|
94
|
+
"""`edited` is a repository doing what it is entitled to do with its own file.
|
|
95
|
+
|
|
96
|
+
`unknown-origin` is a row a newer edify wrote. This version cannot judge it, so
|
|
97
|
+
it says so and moves on rather than failing a build it does not understand.
|
|
98
|
+
"""
|
|
99
|
+
return self.state in ("missing", "modified", "unrecorded")
|
|
100
|
+
|
|
101
|
+
def as_dict(self) -> dict[str, str]:
|
|
102
|
+
return {"path": self.path, "state": self.state, "detail": self.detail}
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
# -- deriving the ledger from the disk ---------------------------------------
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def scan(layout: Layout) -> list[Record]:
|
|
109
|
+
"""What is on the governed surface right now, with each file's attribution.
|
|
110
|
+
|
|
111
|
+
A pure function over files, like everything else in this CLI: run it twice over
|
|
112
|
+
an unchanged tree and it produces the same rows in the same order.
|
|
113
|
+
"""
|
|
114
|
+
records: list[Record] = []
|
|
115
|
+
|
|
116
|
+
for path in sorted(layout.skills_dir.glob("*.md")):
|
|
117
|
+
records.append(_skill_record(layout, path))
|
|
118
|
+
|
|
119
|
+
for path in sorted(layout.commands_dir.glob("*.md")):
|
|
120
|
+
records.append(_shipped_record(layout, path, "command"))
|
|
121
|
+
|
|
122
|
+
for path in sorted(layout.formats_dir.glob("*.md")):
|
|
123
|
+
records.append(_shipped_record(layout, path, "format"))
|
|
124
|
+
|
|
125
|
+
if layout.mcp.is_file():
|
|
126
|
+
records.append(
|
|
127
|
+
Record(
|
|
128
|
+
path=layout.rel(layout.mcp),
|
|
129
|
+
kind="registry",
|
|
130
|
+
origin="seeded",
|
|
131
|
+
provenance="original",
|
|
132
|
+
license=SHIPPED_LICENSE,
|
|
133
|
+
source=f"{DIST_NAME}@{__version__}",
|
|
134
|
+
sha256=sha256_file(layout.mcp),
|
|
135
|
+
edify=__version__,
|
|
136
|
+
)
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
if layout.conventions.is_file():
|
|
140
|
+
records.append(
|
|
141
|
+
Record(
|
|
142
|
+
path=layout.rel(layout.conventions),
|
|
143
|
+
kind="conventions",
|
|
144
|
+
origin="scraped",
|
|
145
|
+
provenance="client",
|
|
146
|
+
license="-",
|
|
147
|
+
source="this repository's own config files",
|
|
148
|
+
sha256=sha256_file(layout.conventions),
|
|
149
|
+
edify=__version__,
|
|
150
|
+
)
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
# The host runtime's mirror — commands, skills, and agents. All three families are
|
|
154
|
+
# generated pointers, so an edit to one is EDIFY's business and `init` will bring it
|
|
155
|
+
# back in line. A file EDIFY did not write is somebody's own and stays out of the
|
|
156
|
+
# ledger; that is what keeps the hand-authored harvest skill from being reported as
|
|
157
|
+
# modified every time anyone touches it, and it is why a hand-written subagent under
|
|
158
|
+
# `.claude/agents/` is left alone the same way.
|
|
159
|
+
for pointers, pattern in layout.host_trees():
|
|
160
|
+
if not pointers.is_dir():
|
|
161
|
+
continue
|
|
162
|
+
for path in sorted(pointers.glob(pattern)):
|
|
163
|
+
if not written_by_edify(path):
|
|
164
|
+
continue
|
|
165
|
+
records.append(
|
|
166
|
+
Record(
|
|
167
|
+
path=layout.rel(path),
|
|
168
|
+
kind="pointer",
|
|
169
|
+
origin="generated",
|
|
170
|
+
provenance="original",
|
|
171
|
+
license=SHIPPED_LICENSE,
|
|
172
|
+
source=f"{DIST_NAME}@{__version__}",
|
|
173
|
+
sha256=sha256_file(path),
|
|
174
|
+
edify=__version__,
|
|
175
|
+
)
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
# Every instruction file any runtime reads — CLAUDE.md, AGENTS.md, GEMINI.md, the
|
|
179
|
+
# Copilot file, the Cursor rule. Hashed as `-` on purpose: EDIFY owns the block
|
|
180
|
+
# between its markers and nothing else in the file, so hashing the whole thing
|
|
181
|
+
# would report every edit somebody makes to their own instructions as a
|
|
182
|
+
# governance event.
|
|
183
|
+
for path in layout.instruction_files():
|
|
184
|
+
if not path.is_file():
|
|
185
|
+
continue
|
|
186
|
+
records.append(
|
|
187
|
+
Record(
|
|
188
|
+
path=layout.rel(path),
|
|
189
|
+
kind="instructions",
|
|
190
|
+
origin="merged",
|
|
191
|
+
provenance="client",
|
|
192
|
+
license="-",
|
|
193
|
+
source=f"{DIST_NAME}@{__version__} owns the edify:begin/end block only",
|
|
194
|
+
sha256="-",
|
|
195
|
+
edify=__version__,
|
|
196
|
+
)
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
if layout.profile.is_file():
|
|
200
|
+
# The one file here nobody derived from anything: a person answered eight
|
|
201
|
+
# questions. Recorded so `list` accounts for it, hashed so an edit shows, and
|
|
202
|
+
# `answered` so nothing EDIFY does later treats it as ours to replace.
|
|
203
|
+
records.append(
|
|
204
|
+
Record(
|
|
205
|
+
path=layout.rel(layout.profile),
|
|
206
|
+
kind="profile",
|
|
207
|
+
origin="answered",
|
|
208
|
+
provenance="client",
|
|
209
|
+
license="-",
|
|
210
|
+
source="answered at `edify init new`",
|
|
211
|
+
sha256=sha256_file(layout.profile),
|
|
212
|
+
edify=__version__,
|
|
213
|
+
)
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
return records
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _skill_record(layout: Layout, path: Path) -> Record:
|
|
220
|
+
parsed = skill_format.parse(path, layout.rel(path))
|
|
221
|
+
# `license: -` in an original entry means "whatever EDIFY ships under". An
|
|
222
|
+
# adapted entry names its upstream licence and that is what governs it.
|
|
223
|
+
declared = parsed.license if parsed.license not in ("", "-") else SHIPPED_LICENSE
|
|
224
|
+
return Record(
|
|
225
|
+
path=layout.rel(path),
|
|
226
|
+
kind=parsed.kind or "skill",
|
|
227
|
+
origin="library",
|
|
228
|
+
provenance=parsed.provenance or "-",
|
|
229
|
+
license=declared,
|
|
230
|
+
source=parsed.source or f"{DIST_NAME}@{__version__}",
|
|
231
|
+
sha256=sha256_file(path),
|
|
232
|
+
edify=__version__,
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _shipped_record(layout: Layout, path: Path, kind: str) -> Record:
|
|
237
|
+
return Record(
|
|
238
|
+
path=layout.rel(path),
|
|
239
|
+
kind=kind,
|
|
240
|
+
origin="shipped",
|
|
241
|
+
provenance="original",
|
|
242
|
+
license=SHIPPED_LICENSE,
|
|
243
|
+
source=f"{DIST_NAME}@{__version__}",
|
|
244
|
+
sha256=sha256_file(path),
|
|
245
|
+
edify=__version__,
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
# -- reading and writing the ledger ------------------------------------------
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def rebuild(layout: Layout) -> list[Record]:
|
|
253
|
+
"""Re-derive the ledger from the disk and write it. The baseline moves to now."""
|
|
254
|
+
records = scan(layout)
|
|
255
|
+
write_rows(layout.governance, [r.row() for r in records], LEDGER_HEADER)
|
|
256
|
+
return records
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def load(layout: Layout) -> list[Record]:
|
|
260
|
+
if not layout.governance.is_file():
|
|
261
|
+
raise EdifyError(
|
|
262
|
+
"no governance ledger",
|
|
263
|
+
hint="run `edify governance rebuild` — or `edify init`, which writes one",
|
|
264
|
+
)
|
|
265
|
+
out: list[Record] = []
|
|
266
|
+
for row in read_rows(layout.governance):
|
|
267
|
+
if len(row) >= len(LEDGER_HEADER):
|
|
268
|
+
out.append(Record(*row[: len(LEDGER_HEADER)]))
|
|
269
|
+
return out
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def verify(layout: Layout) -> list[Problem]:
|
|
273
|
+
"""The ledger against the disk. Three ways they disagree, and they differ in kind."""
|
|
274
|
+
recorded = {r.path: r for r in load(layout)}
|
|
275
|
+
present = {r.path: r for r in scan(layout)}
|
|
276
|
+
problems: list[Problem] = []
|
|
277
|
+
|
|
278
|
+
for path in sorted(set(recorded) | set(present)):
|
|
279
|
+
was = recorded.get(path)
|
|
280
|
+
now = present.get(path)
|
|
281
|
+
if was and was.origin not in ORIGINS:
|
|
282
|
+
problems.append(
|
|
283
|
+
Problem(
|
|
284
|
+
path,
|
|
285
|
+
"unknown-origin",
|
|
286
|
+
f"written by a newer edify (origin `{was.origin}`) — run 0.2.0 or later to read it",
|
|
287
|
+
)
|
|
288
|
+
)
|
|
289
|
+
continue
|
|
290
|
+
if was and not now:
|
|
291
|
+
problems.append(Problem(path, "missing", f"recorded as {was.origin}, not on disk"))
|
|
292
|
+
continue
|
|
293
|
+
if now and not was:
|
|
294
|
+
problems.append(
|
|
295
|
+
Problem(path, "unrecorded", f"on disk as {now.origin}, absent from the ledger")
|
|
296
|
+
)
|
|
297
|
+
continue
|
|
298
|
+
assert was and now
|
|
299
|
+
if was.sha256 in ("-", "") or now.sha256 in ("-", ""):
|
|
300
|
+
continue
|
|
301
|
+
if was.sha256 != now.sha256:
|
|
302
|
+
state = "modified" if was.origin in EDIFY_OWNED else "edited"
|
|
303
|
+
detail = (
|
|
304
|
+
f"changed since it was installed ({was.origin}) — `edify upgrade` will replace it"
|
|
305
|
+
if state == "modified"
|
|
306
|
+
else f"changed since it was written ({was.origin}) — this file is yours to edit"
|
|
307
|
+
)
|
|
308
|
+
problems.append(Problem(path, state, detail))
|
|
309
|
+
return problems
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def summary(records: list[Record]) -> dict[str, int]:
|
|
313
|
+
counts: dict[str, int] = {}
|
|
314
|
+
for record in records:
|
|
315
|
+
counts[record.origin] = counts.get(record.origin, 0) + 1
|
|
316
|
+
return counts
|
edify/graph/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""The map: every file, module, exported symbol, route and table, with edges.
|
|
2
|
+
|
|
3
|
+
Mechanically extracted, never written by a model — not at install, not as a
|
|
4
|
+
fallback, not for a language the extractor does not cover. Where a language is not
|
|
5
|
+
covered the graph has no nodes for it and `meta` says so.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from .model import Edge, Node, EDGE_KINDS, NODE_KINDS
|
|
9
|
+
from .store import GraphStore, load
|
|
10
|
+
|
|
11
|
+
__all__ = ["Edge", "Node", "EDGE_KINDS", "NODE_KINDS", "GraphStore", "load"]
|
edify/graph/extract.py
ADDED
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
"""The extraction pass: walk, extract, resolve, write.
|
|
2
|
+
|
|
3
|
+
No model at any point, for any reason — not at install, not as a fallback, not for
|
|
4
|
+
a language the extractor does not cover. Where a language is not covered the graph
|
|
5
|
+
has no nodes for it and `meta.languages_uncovered` says which.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import subprocess
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from .. import __version__
|
|
16
|
+
from ..errors import EdifyError
|
|
17
|
+
from ..paths import Layout
|
|
18
|
+
from .ignore import IgnoreRules
|
|
19
|
+
from .langs import EXTENSIONS, covered_languages, ctags, extract_file, language_for, method_for
|
|
20
|
+
from .model import Edge, Extraction, Node, node_id
|
|
21
|
+
from .store import GraphStore
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class ExtractResult:
|
|
26
|
+
nodes: int = 0
|
|
27
|
+
edges: int = 0
|
|
28
|
+
files: int = 0
|
|
29
|
+
skipped: int = 0
|
|
30
|
+
languages: dict[str, int] = field(default_factory=dict)
|
|
31
|
+
uncovered: dict[str, int] = field(default_factory=dict)
|
|
32
|
+
backend: str = "builtin"
|
|
33
|
+
truncated_at: int | None = None
|
|
34
|
+
|
|
35
|
+
def as_dict(self) -> dict[str, object]:
|
|
36
|
+
return {
|
|
37
|
+
"nodes": self.nodes,
|
|
38
|
+
"edges": self.edges,
|
|
39
|
+
"files": self.files,
|
|
40
|
+
"skipped": self.skipped,
|
|
41
|
+
"languages": self.languages,
|
|
42
|
+
"uncovered": self.uncovered,
|
|
43
|
+
"backend": self.backend,
|
|
44
|
+
"truncated_at": self.truncated_at,
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def build(
|
|
49
|
+
layout: Layout,
|
|
50
|
+
backend: str = "builtin",
|
|
51
|
+
node_cap: int | None = None,
|
|
52
|
+
subdirs: list[str] | None = None,
|
|
53
|
+
) -> ExtractResult:
|
|
54
|
+
"""Full extraction, or a directory-scoped update when `subdirs` is given."""
|
|
55
|
+
root = layout.root
|
|
56
|
+
if not root.is_dir():
|
|
57
|
+
raise EdifyError(f"{root} is not a directory")
|
|
58
|
+
|
|
59
|
+
rules = IgnoreRules(root)
|
|
60
|
+
scan_roots = _scan_roots(root, subdirs)
|
|
61
|
+
files = _walk(root, scan_roots, rules)
|
|
62
|
+
|
|
63
|
+
result = ExtractResult(backend=backend)
|
|
64
|
+
result.files = len(files)
|
|
65
|
+
|
|
66
|
+
extraction = Extraction()
|
|
67
|
+
if backend == "ctags":
|
|
68
|
+
if ctags.available() is None:
|
|
69
|
+
raise EdifyError(
|
|
70
|
+
"no Universal Ctags on PATH",
|
|
71
|
+
hint="install it, or run without `--extractor ctags` to use the built-in backend",
|
|
72
|
+
)
|
|
73
|
+
extraction.extend(ctags.extract_tree(root, files))
|
|
74
|
+
result.backend = "ctags"
|
|
75
|
+
|
|
76
|
+
for rel in files:
|
|
77
|
+
path = root / rel
|
|
78
|
+
language = language_for(path)
|
|
79
|
+
file_node = Node(node_id("file", rel, "."), "file", rel, 1, rel)
|
|
80
|
+
extraction.nodes.append(file_node)
|
|
81
|
+
|
|
82
|
+
if language is None:
|
|
83
|
+
continue
|
|
84
|
+
if method_for(language) == "none":
|
|
85
|
+
result.uncovered[language] = result.uncovered.get(language, 0) + 1
|
|
86
|
+
continue
|
|
87
|
+
if backend == "ctags" and language != "markdown":
|
|
88
|
+
# The external extractor already covered code. Markdown sections and
|
|
89
|
+
# SQL tables are ours either way — it does not emit them.
|
|
90
|
+
if language != "sql":
|
|
91
|
+
result.languages[language] = result.languages.get(language, 0) + 1
|
|
92
|
+
continue
|
|
93
|
+
|
|
94
|
+
try:
|
|
95
|
+
source = path.read_text(encoding="utf-8", errors="replace")
|
|
96
|
+
except OSError:
|
|
97
|
+
result.skipped += 1
|
|
98
|
+
continue
|
|
99
|
+
|
|
100
|
+
extraction.extend(extract_file(rel, source, language))
|
|
101
|
+
result.languages[language] = result.languages.get(language, 0) + 1
|
|
102
|
+
|
|
103
|
+
_resolve_packages(root, extraction)
|
|
104
|
+
edges = _resolve_refs(extraction)
|
|
105
|
+
|
|
106
|
+
nodes = extraction.nodes
|
|
107
|
+
if node_cap is not None and len(nodes) > node_cap:
|
|
108
|
+
result.truncated_at = node_cap
|
|
109
|
+
keep = {n.id for n in _prioritise(nodes)[:node_cap]}
|
|
110
|
+
nodes = [n for n in nodes if n.id in keep]
|
|
111
|
+
edges = [e for e in edges if e.src in keep and e.dst in keep]
|
|
112
|
+
|
|
113
|
+
store = GraphStore(layout)
|
|
114
|
+
if subdirs:
|
|
115
|
+
nodes, edges = _merge_with_existing(store, nodes, edges, scan_roots, root)
|
|
116
|
+
|
|
117
|
+
meta = _meta(root, result, extraction)
|
|
118
|
+
written = store.write(nodes, edges, meta)
|
|
119
|
+
result.nodes = int(written["nodes"])
|
|
120
|
+
result.edges = int(written["edges"])
|
|
121
|
+
return result
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
# ---------------------------------------------------------------------------
|
|
125
|
+
# the walk
|
|
126
|
+
# ---------------------------------------------------------------------------
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _scan_roots(root: Path, subdirs: list[str] | None) -> list[Path]:
|
|
130
|
+
if not subdirs:
|
|
131
|
+
return [root]
|
|
132
|
+
out: list[Path] = []
|
|
133
|
+
for raw in subdirs:
|
|
134
|
+
candidate = (root / raw).resolve() if not Path(raw).is_absolute() else Path(raw).resolve()
|
|
135
|
+
try:
|
|
136
|
+
candidate.relative_to(root)
|
|
137
|
+
except ValueError:
|
|
138
|
+
raise EdifyError(f"{raw} is outside the repository at {root}") from None
|
|
139
|
+
if not candidate.is_dir():
|
|
140
|
+
raise EdifyError(f"{raw} is not a directory")
|
|
141
|
+
out.append(candidate)
|
|
142
|
+
return out
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _walk(root: Path, scan_roots: list[Path], rules: IgnoreRules) -> list[str]:
|
|
146
|
+
"""Repository-relative POSIX paths, sorted. Sorted because determinism."""
|
|
147
|
+
found: set[str] = set()
|
|
148
|
+
for start in scan_roots:
|
|
149
|
+
for dirpath, dirnames, filenames in os.walk(start):
|
|
150
|
+
here = Path(dirpath)
|
|
151
|
+
rel_dir = here.relative_to(root).as_posix()
|
|
152
|
+
dirnames[:] = sorted(
|
|
153
|
+
d
|
|
154
|
+
for d in dirnames
|
|
155
|
+
if not rules.skip_dir(d, f"{rel_dir}/{d}".lstrip("./") if rel_dir != "." else d)
|
|
156
|
+
)
|
|
157
|
+
for name in filenames:
|
|
158
|
+
rel = (here / name).relative_to(root).as_posix()
|
|
159
|
+
try:
|
|
160
|
+
size = (here / name).stat().st_size
|
|
161
|
+
except OSError:
|
|
162
|
+
continue
|
|
163
|
+
if rules.skip_file(name, rel, size):
|
|
164
|
+
continue
|
|
165
|
+
found.add(rel)
|
|
166
|
+
return sorted(found)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
# ---------------------------------------------------------------------------
|
|
170
|
+
# resolution
|
|
171
|
+
# ---------------------------------------------------------------------------
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _resolve_refs(extraction: Extraction) -> list[Edge]:
|
|
175
|
+
"""Turn pending name references into edges, or drop them.
|
|
176
|
+
|
|
177
|
+
A reference that does not resolve inside the repository is an external one,
|
|
178
|
+
and external things are not in this graph. Dropping is the correct outcome —
|
|
179
|
+
inventing a node for `express` would be narration.
|
|
180
|
+
"""
|
|
181
|
+
by_name: dict[str, list[str]] = {}
|
|
182
|
+
by_module_path: dict[str, str] = {}
|
|
183
|
+
for node in extraction.nodes:
|
|
184
|
+
by_name.setdefault(node.name, []).append(node.id)
|
|
185
|
+
tail = node.name.rsplit(".", 1)[-1]
|
|
186
|
+
if tail != node.name:
|
|
187
|
+
by_name.setdefault(tail, []).append(node.id)
|
|
188
|
+
if node.kind == "module":
|
|
189
|
+
by_module_path[_module_key(node.file)] = node.id
|
|
190
|
+
|
|
191
|
+
# A relative import (`../auth/tokens`) resolves against the importing file's
|
|
192
|
+
# directory, which this pass does not carry. Matching on the path's tail
|
|
193
|
+
# resolves it when exactly one module ends that way, and declines when more
|
|
194
|
+
# than one does — an ambiguous edge is a coin toss, not a fact.
|
|
195
|
+
by_suffix: dict[str, list[str]] = {}
|
|
196
|
+
for key, node_ref in by_module_path.items():
|
|
197
|
+
parts = key.split("/")
|
|
198
|
+
for i in range(len(parts)):
|
|
199
|
+
by_suffix.setdefault("/".join(parts[i:]), []).append(node_ref)
|
|
200
|
+
|
|
201
|
+
edges = list(extraction.edges)
|
|
202
|
+
for src, target, relation in extraction.pending_refs:
|
|
203
|
+
resolved = _lookup(target, by_name, by_module_path, by_suffix)
|
|
204
|
+
if not resolved:
|
|
205
|
+
continue
|
|
206
|
+
for dst in resolved:
|
|
207
|
+
if dst != src:
|
|
208
|
+
edges.append(Edge(src, relation, dst))
|
|
209
|
+
return edges
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _lookup(
|
|
213
|
+
target: str,
|
|
214
|
+
by_name: dict[str, list[str]],
|
|
215
|
+
by_module_path: dict[str, str],
|
|
216
|
+
by_suffix: dict[str, list[str]],
|
|
217
|
+
) -> list[str]:
|
|
218
|
+
# A path-like import: './auth/session', 'src/auth/session', 'app.auth.session'
|
|
219
|
+
cleaned = target.strip().lstrip("./").replace("\\", "/")
|
|
220
|
+
for candidate in (cleaned, cleaned.replace(".", "/")):
|
|
221
|
+
key = _module_key(candidate)
|
|
222
|
+
if key in by_module_path:
|
|
223
|
+
return [by_module_path[key]]
|
|
224
|
+
suffixed = by_suffix.get(key, [])
|
|
225
|
+
if len(suffixed) == 1:
|
|
226
|
+
return suffixed
|
|
227
|
+
# A bare name: resolve only when it is unambiguous. Two symbols with the same
|
|
228
|
+
# name in different files is not an edge, it is a coin toss.
|
|
229
|
+
hits = by_name.get(target) or by_name.get(target.rsplit(".", 1)[-1]) or []
|
|
230
|
+
return hits if len(hits) == 1 else []
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _module_key(path: str) -> str:
|
|
234
|
+
key = path.strip().strip("/")
|
|
235
|
+
for suffix in (
|
|
236
|
+
".py", ".pyi", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".go", ".rs",
|
|
237
|
+
".java", ".kt", ".rb", ".php", ".cs", ".swift",
|
|
238
|
+
):
|
|
239
|
+
if key.endswith(suffix):
|
|
240
|
+
key = key[: -len(suffix)]
|
|
241
|
+
break
|
|
242
|
+
if key.endswith("/index"):
|
|
243
|
+
key = key[: -len("/index")]
|
|
244
|
+
if key.endswith("/__init__"):
|
|
245
|
+
key = key[: -len("/__init__")]
|
|
246
|
+
if key.endswith("/mod"):
|
|
247
|
+
key = key[: -len("/mod")]
|
|
248
|
+
return key
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _resolve_packages(root: Path, extraction: Extraction) -> None:
|
|
252
|
+
"""Declared dependencies become `package` nodes.
|
|
253
|
+
|
|
254
|
+
Read from manifests only — never from an import statement, because an import
|
|
255
|
+
of something that is not declared is a fact about the code, not a package.
|
|
256
|
+
"""
|
|
257
|
+
from ..detect import declared_packages
|
|
258
|
+
|
|
259
|
+
for name, manifest in declared_packages(root).items():
|
|
260
|
+
pkg_id = node_id("package", manifest, name)
|
|
261
|
+
extraction.nodes.append(Node(pkg_id, "package", manifest, 1, name))
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _prioritise(nodes: list[Node]) -> list[Node]:
|
|
265
|
+
"""Ordering used only when a node cap truncates the graph.
|
|
266
|
+
|
|
267
|
+
Keep the public surface first: the map is worth more with every exported
|
|
268
|
+
symbol and half the file nodes than the other way round.
|
|
269
|
+
"""
|
|
270
|
+
rank = {"symbol": 0, "route": 1, "table": 2, "module": 3, "package": 4, "file": 5, "doc-section": 6}
|
|
271
|
+
return sorted(nodes, key=lambda n: (rank.get(n.kind, 9), n.file, n.line, n.name))
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _merge_with_existing(
|
|
275
|
+
store: GraphStore, nodes: list[Node], edges: list[Edge], scan_roots: list[Path], root: Path
|
|
276
|
+
) -> tuple[list[Node], list[Edge]]:
|
|
277
|
+
"""A directory update replaces that directory's nodes and keeps the rest."""
|
|
278
|
+
if not store.exists():
|
|
279
|
+
return nodes, edges
|
|
280
|
+
prefixes = tuple(
|
|
281
|
+
(r.relative_to(root).as_posix().rstrip("/") + "/") if r != root else "" for r in scan_roots
|
|
282
|
+
)
|
|
283
|
+
if "" in prefixes:
|
|
284
|
+
return nodes, edges
|
|
285
|
+
|
|
286
|
+
kept_nodes = [n for n in store.nodes if not n.file.startswith(prefixes)]
|
|
287
|
+
kept_ids = {n.id for n in kept_nodes} | {n.id for n in nodes}
|
|
288
|
+
kept_edges = [
|
|
289
|
+
e for e in store.edges if e.src in kept_ids and e.dst in kept_ids and _outside(e, store, prefixes)
|
|
290
|
+
]
|
|
291
|
+
return kept_nodes + nodes, kept_edges + edges
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _outside(edge: Edge, store: GraphStore, prefixes: tuple[str, ...]) -> bool:
|
|
295
|
+
src = store.by_id.get(edge.src)
|
|
296
|
+
return not (src and src.file.startswith(prefixes))
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
# ---------------------------------------------------------------------------
|
|
300
|
+
# meta
|
|
301
|
+
# ---------------------------------------------------------------------------
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _meta(root: Path, result: ExtractResult, extraction: Extraction) -> dict[str, str]:
|
|
305
|
+
covered = sorted(result.languages)
|
|
306
|
+
meta = {
|
|
307
|
+
"commit": _head_commit(root),
|
|
308
|
+
"extractor": result.backend,
|
|
309
|
+
"extractor_version": ctags.version() if result.backend == "ctags" else f"edify {__version__}",
|
|
310
|
+
"languages_covered": " ".join(covered) or "-",
|
|
311
|
+
"languages_uncovered": " ".join(sorted(result.uncovered)) or "-",
|
|
312
|
+
"files_scanned": str(result.files),
|
|
313
|
+
}
|
|
314
|
+
for language in covered:
|
|
315
|
+
meta[f"method_{language}"] = (
|
|
316
|
+
"external" if result.backend == "ctags" and language not in ("markdown", "sql")
|
|
317
|
+
else method_for(language)
|
|
318
|
+
)
|
|
319
|
+
if result.truncated_at is not None:
|
|
320
|
+
meta["truncated_at"] = str(result.truncated_at)
|
|
321
|
+
return meta
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _head_commit(root: Path) -> str:
|
|
325
|
+
"""The commit the graph was built from, or `-` where there is no git.
|
|
326
|
+
|
|
327
|
+
`-` is honest and the freshness check reads it as "cannot tell". A fabricated
|
|
328
|
+
value here would make a stale graph look current, which is the failure mode
|
|
329
|
+
this field exists to prevent.
|
|
330
|
+
"""
|
|
331
|
+
try:
|
|
332
|
+
out = subprocess.run(
|
|
333
|
+
["git", "rev-parse", "HEAD"],
|
|
334
|
+
cwd=str(root),
|
|
335
|
+
capture_output=True,
|
|
336
|
+
text=True,
|
|
337
|
+
timeout=10,
|
|
338
|
+
check=False,
|
|
339
|
+
)
|
|
340
|
+
except (OSError, subprocess.SubprocessError):
|
|
341
|
+
return "-"
|
|
342
|
+
value = out.stdout.strip()
|
|
343
|
+
return value if out.returncode == 0 and value else "-"
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def known_extensions() -> list[str]:
|
|
347
|
+
return sorted(EXTENSIONS)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def known_languages() -> list[str]:
|
|
351
|
+
return covered_languages()
|